merge from trunk

15 files changed, 1346 insertions, 402 deletions

diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py
index 56a36906..4a3b21af 100644
--- a/cloudinit/config/cc_set_passwords.py
+++ b/cloudinit/config/cc_set_passwords.py
@@ -136,9 +136,12 @@ def handle(_name, cfg, cloud, log, args):
         util.write_file(ssh_util.DEF_SSHD_CFG, "\n".join(lines))
 
         try:
-            cmd = ['service']
+            cmd = cloud.distro.init_cmd  # Default service
             cmd.append(cloud.distro.get_option('ssh_svcname', 'ssh'))
             cmd.append('restart')
+            if 'systemctl' in cmd:  # Switch action ordering
+                cmd[1], cmd[2] = cmd[2], cmd[1]
+            cmd = filter(None, cmd)  # Remove empty arguments
             util.subp(cmd)
             log.debug("Restarted the ssh daemon")
         except:
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 46b67fa3..55d6bcbc 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -39,8 +39,10 @@ from cloudinit.distros.parsers import hosts
 OSFAMILIES = {
     'debian': ['debian', 'ubuntu'],
     'redhat': ['fedora', 'rhel'],
+    'gentoo': ['gentoo'],
     'freebsd': ['freebsd'],
-    'suse': ['sles']
+    'suse': ['sles'],
+    'arch': ['arch'],
 }
 
 LOG = logging.getLogger(__name__)
@@ -53,6 +55,7 @@ class Distro(object):
     ci_sudoers_fn = "/etc/sudoers.d/90-cloud-init-users"
     hostname_conf_fn = "/etc/hostname"
     tz_zone_dir = "/usr/share/zoneinfo"
+    init_cmd = ['service']  # systemctl, service etc
 
     def __init__(self, name, cfg, paths):
         self._paths = paths
diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py
new file mode 100644
index 00000000..310c3dff
--- /dev/null
+++ b/cloudinit/distros/arch.py
@@ -0,0 +1,219 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2014 Rackspace, US Inc.
+#
+#    Author: Nate House <nathan.house@rackspace.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from cloudinit import distros
+from cloudinit import helpers
+from cloudinit import log as logging
+from cloudinit import util
+
+from cloudinit.distros import net_util
+from cloudinit.distros.parsers.hostname import HostnameConf
+
+from cloudinit.settings import PER_INSTANCE
+
+LOG = logging.getLogger(__name__)
+
+
+class Distro(distros.Distro):
+    locale_conf_fn = "/etc/locale.gen"
+    network_conf_dir = "/etc/netctl"
+    tz_conf_fn = "/etc/timezone"
+    tz_local_fn = "/etc/localtime"
+    resolve_conf_fn = "/etc/resolv.conf"
+    init_cmd = ['systemctl']  # init scripts
+
+    def __init__(self, name, cfg, paths):
+        distros.Distro.__init__(self, name, cfg, paths)
+        # This will be used to restrict certain
+        # calls from repeatly happening (when they
+        # should only happen say once per instance...)
+        self._runner = helpers.Runners(paths)
+        self.osfamily = 'arch'
+        cfg['ssh_svcname'] = 'sshd'
+
+    def apply_locale(self, locale, out_fn=None):
+        if not out_fn:
+            out_fn = self.locale_conf_fn
+        util.subp(['locale-gen', '-G', locale], capture=False)
+        # "" provides trailing newline during join
+        lines = [
+            util.make_header(),
+            'LANG="%s"' % (locale),
+            "",
+        ]
+        util.write_file(out_fn, "\n".join(lines))
+
+    def install_packages(self, pkglist):
+        self.update_package_sources()
+        self.package_command('', pkgs=pkglist)
+
+    def _write_network(self, settings):
+        entries = net_util.translate_network(settings)
+        LOG.debug("Translated ubuntu style network settings %s into %s",
+                  settings, entries)
+        dev_names = entries.keys()
+        # Format for netctl
+        for (dev, info) in entries.iteritems():
+            nameservers = []
+            net_fn = self.network_conf_dir + dev
+            net_cfg = {
+                'Connection': 'ethernet',
+                'Interface': dev,
+                'IP': info.get('bootproto'),
+                'Address': "('%s/%s')" % (info.get('address'),
+                        info.get('netmask')),
+                'Gateway': info.get('gateway'),
+                'DNS': str(tuple(info.get('dns-nameservers'))).replace(',', '')
+            }
+            util.write_file(net_fn, convert_netctl(net_cfg))
+            if info.get('auto'):
+                self._enable_interface(dev)
+            if 'dns-nameservers' in info:
+                nameservers.extend(info['dns-nameservers'])
+
+        if nameservers:
+            util.write_file(self.resolve_conf_fn,
+                    convert_resolv_conf(nameservers))
+
+        return dev_names
+
+    def _enable_interface(self, device_name):
+        cmd = ['netctl', 'reenable', device_name]
+        try:
+            (_out, err) = util.subp(cmd)
+            if len(err):
+                LOG.warn("Running %s resulted in stderr output: %s", cmd, err)
+        except util.ProcessExecutionError:
+            util.logexc(LOG, "Running interface command %s failed", cmd)
+
+    def _bring_up_interface(self, device_name):
+        cmd = ['netctl', 'restart', device_name]
+        LOG.debug("Attempting to run bring up interface %s using command %s",
+                   device_name, cmd)
+        try:
+            (_out, err) = util.subp(cmd)
+            if len(err):
+                LOG.warn("Running %s resulted in stderr output: %s", cmd, err)
+            return True
+        except util.ProcessExecutionError:
+            util.logexc(LOG, "Running interface command %s failed", cmd)
+            return False
+
+    def _bring_up_interfaces(self, device_names):
+        for d in device_names:
+            if not self._bring_up_interface(d):
+                return False
+        return True
+
+    def _select_hostname(self, hostname, fqdn):
+        # Prefer the short hostname over the long
+        # fully qualified domain name
+        if not hostname:
+            return fqdn
+        return hostname
+
+    def _write_hostname(self, your_hostname, out_fn):
+        conf = None
+        try:
+            # Try to update the previous one
+            # so lets see if we can read it first.
+            conf = self._read_hostname_conf(out_fn)
+        except IOError:
+            pass
+        if not conf:
+            conf = HostnameConf('')
+        conf.set_hostname(your_hostname)
+        util.write_file(out_fn, str(conf), 0644)
+
+    def _read_system_hostname(self):
+        sys_hostname = self._read_hostname(self.hostname_conf_fn)
+        return (self.hostname_conf_fn, sys_hostname)
+
+    def _read_hostname_conf(self, filename):
+        conf = HostnameConf(util.load_file(filename))
+        conf.parse()
+        return conf
+
+    def _read_hostname(self, filename, default=None):
+        hostname = None
+        try:
+            conf = self._read_hostname_conf(filename)
+            hostname = conf.hostname
+        except IOError:
+            pass
+        if not hostname:
+            return default
+        return hostname
+
+    def set_timezone(self, tz):
+        tz_file = self._find_tz_file(tz)
+        # Note: "" provides trailing newline during join
+        tz_lines = [
+            util.make_header(),
+            str(tz),
+            "",
+        ]
+        util.write_file(self.tz_conf_fn, "\n".join(tz_lines))
+        # This ensures that the correct tz will be used for the system
+        util.copy(tz_file, self.tz_local_fn)
+
+    def package_command(self, command, args=None, pkgs=None):
+        if pkgs is None:
+            pkgs = []
+
+        cmd = ['pacman']
+        # Redirect output
+        cmd.append("-Sy")
+        cmd.append("--quiet")
+        cmd.append("--noconfirm")
+
+        if args and isinstance(args, str):
+            cmd.append(args)
+        elif args and isinstance(args, list):
+            cmd.extend(args)
+
+        if command:
+            cmd.append(command)
+
+        pkglist = util.expand_package_list('%s-%s', pkgs)
+        cmd.extend(pkglist)
+
+        # Allow the output of this to flow outwards (ie not be captured)
+        util.subp(cmd, capture=False)
+
+    def update_package_sources(self):
+        self._runner.run("update-sources", self.package_command,
+                         ["-y"], freq=PER_INSTANCE)
+
+
+def convert_netctl(settings):
+    """Returns a settings string formatted for netctl."""
+    result = ''
+    if isinstance(settings, dict):
+        for k, v in settings.items():
+            result = result + '%s=%s\n' % (k, v)
+        return result
+
+
+def convert_resolv_conf(settings):
+    """Returns a settings string formatted for resolv.conf."""
+    result = ''
+    if isinstance(settings, list):
+        for ns in list:
+            result = result + 'nameserver %s\n' % ns
+        return result
diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py
new file mode 100644
index 00000000..09f8d8ea
--- /dev/null
+++ b/cloudinit/distros/gentoo.py
@@ -0,0 +1,178 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2014 Rackspace, US Inc.
+#
+#    Author: Nate House <nathan.house@rackspace.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from cloudinit import distros
+from cloudinit import helpers
+from cloudinit import log as logging
+from cloudinit import util
+
+from cloudinit.distros.parsers.hostname import HostnameConf
+
+from cloudinit.settings import PER_INSTANCE
+
+LOG = logging.getLogger(__name__)
+
+
+class Distro(distros.Distro):
+    locale_conf_fn = "/etc/locale.gen"
+    network_conf_fn = "/etc/conf.d/net"
+    tz_conf_fn = "/etc/timezone"
+    tz_local_fn = "/etc/localtime"
+    init_cmd = ['']  # init scripts
+
+    def __init__(self, name, cfg, paths):
+        distros.Distro.__init__(self, name, cfg, paths)
+        # This will be used to restrict certain
+        # calls from repeatly happening (when they
+        # should only happen say once per instance...)
+        self._runner = helpers.Runners(paths)
+        self.osfamily = 'gentoo'
+        # Fix sshd restarts
+        cfg['ssh_svcname'] = '/etc/init.d/sshd'
+
+    def apply_locale(self, locale, out_fn=None):
+        if not out_fn:
+            out_fn = self.locale_conf_fn
+        util.subp(['locale-gen', '-G', locale], capture=False)
+        # "" provides trailing newline during join
+        lines = [
+            util.make_header(),
+            'LANG="%s"' % (locale),
+            "",
+        ]
+        util.write_file(out_fn, "\n".join(lines))
+
+    def install_packages(self, pkglist):
+        self.update_package_sources()
+        self.package_command('', pkgs=pkglist)
+
+    def _write_network(self, settings):
+        util.write_file(self.network_conf_fn, settings)
+        return ['all']
+
+    def _bring_up_interface(self, device_name):
+        cmd = ['/etc/init.d/net.%s' % device_name, 'restart']
+        LOG.debug("Attempting to run bring up interface %s using command %s",
+                   device_name, cmd)
+        try:
+            (_out, err) = util.subp(cmd)
+            if len(err):
+                LOG.warn("Running %s resulted in stderr output: %s", cmd, err)
+            return True
+        except util.ProcessExecutionError:
+            util.logexc(LOG, "Running interface command %s failed", cmd)
+            return False
+
+    def _bring_up_interfaces(self, device_names):
+        use_all = False
+        for d in device_names:
+            if d == 'all':
+                use_all = True
+        if use_all:
+            # Grab device names from init scripts
+            cmd = ['ls', '/etc/init.d/net.*']
+            try:
+                (_out, err) = util.subp(cmd)
+                if len(err):
+                    LOG.warn("Running %s resulted in stderr output: %s", cmd,
+                            err)
+            except util.ProcessExecutionError:
+                util.logexc(LOG, "Running interface command %s failed", cmd)
+                return False
+            devices = [x.split('.')[2] for x in _out.split('  ')]
+            return distros.Distro._bring_up_interfaces(self, devices)
+        else:
+            return distros.Distro._bring_up_interfaces(self, device_names)
+
+    def _select_hostname(self, hostname, fqdn):
+        # Prefer the short hostname over the long
+        # fully qualified domain name
+        if not hostname:
+            return fqdn
+        return hostname
+
+    def _write_hostname(self, your_hostname, out_fn):
+        conf = None
+        try:
+            # Try to update the previous one
+            # so lets see if we can read it first.
+            conf = self._read_hostname_conf(out_fn)
+        except IOError:
+            pass
+        if not conf:
+            conf = HostnameConf('')
+        conf.set_hostname(your_hostname)
+        util.write_file(out_fn, str(conf), 0644)
+
+    def _read_system_hostname(self):
+        sys_hostname = self._read_hostname(self.hostname_conf_fn)
+        return (self.hostname_conf_fn, sys_hostname)
+
+    def _read_hostname_conf(self, filename):
+        conf = HostnameConf(util.load_file(filename))
+        conf.parse()
+        return conf
+
+    def _read_hostname(self, filename, default=None):
+        hostname = None
+        try:
+            conf = self._read_hostname_conf(filename)
+            hostname = conf.hostname
+        except IOError:
+            pass
+        if not hostname:
+            return default
+        return hostname
+
+    def set_timezone(self, tz):
+        tz_file = self._find_tz_file(tz)
+        # Note: "" provides trailing newline during join
+        tz_lines = [
+            util.make_header(),
+            str(tz),
+            "",
+        ]
+        util.write_file(self.tz_conf_fn, "\n".join(tz_lines))
+        # This ensures that the correct tz will be used for the system
+        util.copy(tz_file, self.tz_local_fn)
+
+    def package_command(self, command, args=None, pkgs=None):
+        if pkgs is None:
+            pkgs = []
+
+        cmd = ['emerge']
+        # Redirect output
+        cmd.append("--quiet")
+
+        if args and isinstance(args, str):
+            cmd.append(args)
+        elif args and isinstance(args, list):
+            cmd.extend(args)
+
+        if command:
+            cmd.append(command)
+
+        pkglist = util.expand_package_list('%s-%s', pkgs)
+        cmd.extend(pkglist)
+
+        # Allow the output of this to flow outwards (ie not be captured)
+        util.subp(cmd, capture=False)
+
+    def update_package_sources(self):
+        self._runner.run("update-sources", self.package_command,
+                         ["-u", "world"], freq=PER_INSTANCE)
diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
index 7f4c0443..a7c9c9ab 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/ec2_utils.py
@@ -16,12 +16,9 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import httplib
-from urlparse import (urlparse, urlunparse)
-
 import functools
+import httplib
 import json
-import urllib
 
 from cloudinit import log as logging
 from cloudinit import url_helper
@@ -40,16 +37,6 @@ def maybe_json_object(text):
     return False
 
 
-def combine_url(base, add_on):
-    base_parsed = list(urlparse(base))
-    path = base_parsed[2]
-    if path and not path.endswith("/"):
-        path += "/"
-    path += urllib.quote(str(add_on), safe="/:")
-    base_parsed[2] = path
-    return urlunparse(base_parsed)
-
-
 # See: http://bit.ly/TyoUQs
 #
 class MetadataMaterializer(object):
@@ -121,14 +108,14 @@ class MetadataMaterializer(object):
         (leaves, children) = self._parse(blob)
         child_contents = {}
         for c in children:
-            child_url = combine_url(base_url, c)
+            child_url = url_helper.combine_url(base_url, c)
             if not child_url.endswith("/"):
                 child_url += "/"
             child_blob = str(self._caller(child_url))
             child_contents[c] = self._materialize(child_blob, child_url)
         leaf_contents = {}
         for (field, resource) in leaves.items():
-            leaf_url = combine_url(base_url, resource)
+            leaf_url = url_helper.combine_url(base_url, resource)
             leaf_blob = str(self._caller(leaf_url))
             leaf_contents[field] = self._decode_leaf_blob(field, leaf_blob)
         joined = {}
@@ -153,8 +140,8 @@ def _skip_retry_on_codes(status_codes, _request_args, cause):
 def get_instance_userdata(api_version='latest',
                           metadata_address='http://169.254.169.254',
                           ssl_details=None, timeout=5, retries=5):
-    ud_url = combine_url(metadata_address, api_version)
-    ud_url = combine_url(ud_url, 'user-data')
+    ud_url = url_helper.combine_url(metadata_address, api_version)
+    ud_url = url_helper.combine_url(ud_url, 'user-data')
     user_data = ''
     try:
         # It is ok for userdata to not exist (thats why we are stopping if
@@ -178,8 +165,8 @@ def get_instance_userdata(api_version='latest',
 def get_instance_metadata(api_version='latest',
                           metadata_address='http://169.254.169.254',
                           ssl_details=None, timeout=5, retries=5):
-    md_url = combine_url(metadata_address, api_version)
-    md_url = combine_url(md_url, 'meta-data')
+    md_url = url_helper.combine_url(metadata_address, api_version)
+    md_url = url_helper.combine_url(md_url, 'meta-data')
     caller = functools.partial(util.read_file_or_url,
                                ssl_details=ssl_details, timeout=timeout,
                                retries=retries)
diff --git a/cloudinit/settings.py b/cloudinit/settings.py
index 7b0b18e7..37d4958b 100644
--- a/cloudinit/settings.py
+++ b/cloudinit/settings.py
@@ -36,6 +36,8 @@ CFG_BUILTIN = {
         'AltCloud',
         'OVF',
         'MAAS',
+        'GCE',
+        'OpenStack'
         'Ec2',
         'CloudSigma',
         'CloudStack',
diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py
index 78acd8a4..e734d7e5 100644
--- a/cloudinit/sources/DataSourceCloudSigma.py
+++ b/cloudinit/sources/DataSourceCloudSigma.py
@@ -45,18 +45,25 @@ class DataSourceCloudSigma(sources.DataSource):
         Metadata is the whole server context and /meta/cloud-config is used
         as userdata.
         """
+        dsmode = None
         try:
             server_context = self.cepko.all().result
             server_meta = server_context['meta']
-            self.userdata_raw = server_meta.get('cloudinit-user-data', "")
-            self.metadata = server_context
-            self.ssh_public_key = server_meta['ssh_public_key']
-
-            if server_meta.get('cloudinit-dsmode') in VALID_DSMODES:
-                self.dsmode = server_meta['cloudinit-dsmode']
         except:
             util.logexc(LOG, "Failed reading from the serial port")
             return False
+
+        dsmode = server_meta.get('cloudinit-dsmode', self.dsmode)
+        if dsmode not in VALID_DSMODES:
+            LOG.warn("Invalid dsmode %s, assuming default of 'net'", dsmode)
+            dsmode = 'net'
+        if dsmode == "disabled" or dsmode != self.dsmode:
+            return False
+
+        self.userdata_raw = server_meta.get('cloudinit-user-data', "")
+        self.metadata = server_context
+        self.ssh_public_key = server_meta['ssh_public_key']
+
         return True
 
     def get_hostname(self, fqdn=False, resolve_ip=False):
@@ -76,11 +83,17 @@ class DataSourceCloudSigma(sources.DataSource):
         return self.metadata['uuid']
 
 
+class DataSourceCloudSigmaNet(DataSourceCloudSigma):
+    def __init__(self, sys_cfg, distro, paths):
+        DataSourceCloudSigma.__init__(self, sys_cfg, distro, paths)
+        self.dsmode = 'net'
+
+
 # Used to match classes to dependencies. Since this datasource uses the serial
 # port network is not really required, so it's okay to load without it, too.
 datasources = [
     (DataSourceCloudSigma, (sources.DEP_FILESYSTEM)),
-    (DataSourceCloudSigma, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+    (DataSourceCloudSigmaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
 ]
 
 
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 2a244496..142e0eb8 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -18,181 +18,79 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import base64
-import json
 import os
 
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import util
 
+from cloudinit.sources.helpers import openstack
+
 LOG = logging.getLogger(__name__)
 
 # Various defaults/constants...
 DEFAULT_IID = "iid-dsconfigdrive"
 DEFAULT_MODE = 'pass'
-CFG_DRIVE_FILES_V1 = [
-    "etc/network/interfaces",
-    "root/.ssh/authorized_keys",
-    "meta.js",
-]
 DEFAULT_METADATA = {
     "instance-id": DEFAULT_IID,
 }
 VALID_DSMODES = ("local", "net", "pass", "disabled")
+FS_TYPES = ('vfat', 'iso9660')
+LABEL_TYPES = ('config-2',)
+OPTICAL_DEVICES = tuple(('/dev/sr%s' % i for i in range(0, 2)))
 
 
-class ConfigDriveHelper(object):
-    def __init__(self, distro):
-        self.distro = distro
-
-    def on_first_boot(self, data):
-        if not data:
-            data = {}
-        if 'network_config' in data:
-            LOG.debug("Updating network interfaces from config drive")
-            self.distro.apply_network(data['network_config'])
-        files = data.get('files')
-        if files:
-            LOG.debug("Writing %s injected files", len(files))
-            try:
-                write_files(files)
-            except IOError:
-                util.logexc(LOG, "Failed writing files")
-
-
-class DataSourceConfigDrive(sources.DataSource):
+class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
-        sources.DataSource.__init__(self, sys_cfg, distro, paths)
+        super(DataSourceConfigDrive, self).__init__(sys_cfg, distro, paths)
         self.source = None
         self.dsmode = 'local'
         self.seed_dir = os.path.join(paths.seed_dir, 'config_drive')
         self.version = None
         self.ec2_metadata = None
-        self.helper = ConfigDriveHelper(distro)
+        self.files = {}
 
     def __str__(self):
         root = sources.DataSource.__str__(self)
-        mstr = "%s [%s,ver=%s]" % (root,
-                                   self.dsmode,
-                                   self.version)
+        mstr = "%s [%s,ver=%s]" % (root, self.dsmode, self.version)
         mstr += "[source=%s]" % (self.source)
         return mstr
 
-    def _ec2_name_to_device(self, name):
-        if not self.ec2_metadata:
-            return None
-        bdm = self.ec2_metadata.get('block-device-mapping', {})
-        for (ent_name, device) in bdm.items():
-            if name == ent_name:
-                return device
-        return None
-
-    def _os_name_to_device(self, name):
-        device = None
-        try:
-            criteria = 'LABEL=%s' % (name)
-            if name in ['swap']:
-                criteria = 'TYPE=%s' % (name)
-            dev_entries = util.find_devs_with(criteria)
-            if dev_entries:
-                device = dev_entries[0]
-        except util.ProcessExecutionError:
-            pass
-        return device
-
-    def _validate_device_name(self, device):
-        if not device:
-            return None
-        if not device.startswith("/"):
-            device = "/dev/%s" % device
-        if os.path.exists(device):
-            return device
-        # Durn, try adjusting the mapping
-        remapped = self._remap_device(os.path.basename(device))
-        if remapped:
-            LOG.debug("Remapped device name %s => %s", device, remapped)
-            return remapped
-        return None
-
-    def device_name_to_device(self, name):
-        # Translate a 'name' to a 'physical' device
-        if not name:
-            return None
-        # Try the ec2 mapping first
-        names = [name]
-        if name == 'root':
-            names.insert(0, 'ami')
-        if name == 'ami':
-            names.append('root')
-        device = None
-        LOG.debug("Using ec2 metadata lookup to find device %s", names)
-        for n in names:
-            device = self._ec2_name_to_device(n)
-            device = self._validate_device_name(device)
-            if device:
-                break
-        # Try the openstack way second
-        if not device:
-            LOG.debug("Using os lookup to find device %s", names)
-            for n in names:
-                device = self._os_name_to_device(n)
-                device = self._validate_device_name(device)
-                if device:
-                    break
-        # Ok give up...
-        if not device:
-            return None
-        else:
-            LOG.debug("Using cfg drive lookup mapped to device %s", device)
-            return device
-
     def get_data(self):
         found = None
         md = {}
-
         results = {}
         if os.path.isdir(self.seed_dir):
             try:
-                results = read_config_drive_dir(self.seed_dir)
+                results = read_config_drive(self.seed_dir)
                 found = self.seed_dir
-            except NonConfigDriveDir:
+            except openstack.NonReadable:
                 util.logexc(LOG, "Failed reading config drive from %s",
                             self.seed_dir)
         if not found:
-            devlist = find_candidate_devs()
-            for dev in devlist:
+            for dev in find_candidate_devs():
                 try:
-                    results = util.mount_cb(dev, read_config_drive_dir)
+                    results = util.mount_cb(dev, read_config_drive)
                     found = dev
-                    break
-                except (NonConfigDriveDir, util.MountFailedError):
+                except openstack.NonReadable:
                     pass
-                except BrokenConfigDriveDir:
-                    util.logexc(LOG, "broken config drive: %s", dev)
-
+                except util.MountFailedError:
+                    pass
+                except openstack.BrokenMetadata:
+                    util.logexc(LOG, "Broken config drive: %s", dev)
+                if found:
+                    break
         if not found:
             return False
 
-        md = results['metadata']
+        md = results.get('metadata', {})
         md = util.mergemanydict([md, DEFAULT_METADATA])
-
-        # Perform some metadata 'fixups'
-        #
-        # OpenStack uses the 'hostname' key
-        # while most of cloud-init uses the metadata
-        # 'local-hostname' key instead so if it doesn't
-        # exist we need to make sure its copied over.
-        for (tgt, src) in [('local-hostname', 'hostname')]:
-            if tgt not in md and src in md:
-                md[tgt] = md[src]
-
         user_dsmode = results.get('dsmode', None)
         if user_dsmode not in VALID_DSMODES + (None,):
-            LOG.warn("user specified invalid mode: %s" % user_dsmode)
+            LOG.warn("User specified invalid mode: %s", user_dsmode)
             user_dsmode = None
 
-        dsmode = get_ds_mode(cfgdrv_ver=results['cfgdrive_ver'],
+        dsmode = get_ds_mode(cfgdrv_ver=results['version'],
                              ds_cfg=self.ds_cfg.get('dsmode'),
                              user=user_dsmode)
 
@@ -209,7 +107,7 @@ class DataSourceConfigDrive(sources.DataSource):
         prev_iid = get_previous_iid(self.paths)
         cur_iid = md['instance-id']
         if prev_iid != cur_iid and self.dsmode == "local":
-            self.helper.on_first_boot(results)
+            on_first_boot(results, distro=self.distro)
 
         # dsmode != self.dsmode here if:
         #  * dsmode = "pass",  pass means it should only copy files and then
@@ -225,16 +123,11 @@ class DataSourceConfigDrive(sources.DataSource):
         self.metadata = md
         self.ec2_metadata = results.get('ec2-metadata')
         self.userdata_raw = results.get('userdata')
-        self.version = results['cfgdrive_ver']
-
+        self.version = results['version']
+        self.files.update(results.get('files', {}))
+        self.vendordata_raw = results.get('vendordata')
         return True
 
-    def get_public_ssh_keys(self):
-        name = "public_keys"
-        if self.version == 1:
-            name = "public-keys"
-        return sources.normalize_pubkey_data(self.metadata.get(name))
-
 
 class DataSourceConfigDriveNet(DataSourceConfigDrive):
     def __init__(self, sys_cfg, distro, paths):
@@ -242,222 +135,6 @@ class DataSourceConfigDriveNet(DataSourceConfigDrive):
         self.dsmode = 'net'
 
 
-class NonConfigDriveDir(Exception):
-    pass
-
-
-class BrokenConfigDriveDir(Exception):
-    pass
-
-
-def find_candidate_devs():
-    """Return a list of devices that may contain the config drive.
-
-    The returned list is sorted by search order where the first item has
-    should be searched first (highest priority)
-
-    config drive v1:
-       Per documentation, this is "associated as the last available disk on the
-       instance", and should be VFAT.
-       Currently, we do not restrict search list to "last available disk"
-
-    config drive v2:
-       Disk should be:
-        * either vfat or iso9660 formated
-        * labeled with 'config-2'
-    """
-
-    # Query optical drive to get it in blkid cache for 2.6 kernels
-    util.find_devs_with(path="/dev/sr0")
-    util.find_devs_with(path="/dev/sr1")
-
-    by_fstype = (util.find_devs_with("TYPE=vfat") +
-                 util.find_devs_with("TYPE=iso9660"))
-    by_label = util.find_devs_with("LABEL=config-2")
-
-    # give preference to "last available disk" (vdb over vda)
-    # note, this is not a perfect rendition of that.
-    by_fstype.sort(reverse=True)
-    by_label.sort(reverse=True)
-
-    # combine list of items by putting by-label items first
-    # followed by fstype items, but with dupes removed
-    combined = (by_label + [d for d in by_fstype if d not in by_label])
-
-    # We are looking for a block device or partition with necessary label or
-    # an unpartitioned block device.
-    combined = [d for d in combined
-                    if d in by_label or not util.is_partition(d)]
-
-    return combined
-
-
-def read_config_drive_dir(source_dir):
-    last_e = NonConfigDriveDir("Not found")
-    for finder in (read_config_drive_dir_v2, read_config_drive_dir_v1):
-        try:
-            data = finder(source_dir)
-            return data
-        except NonConfigDriveDir as exc:
-            last_e = exc
-    raise last_e
-
-
-def read_config_drive_dir_v2(source_dir, version="2012-08-10"):
-
-    if (not os.path.isdir(os.path.join(source_dir, "openstack", version)) and
-        os.path.isdir(os.path.join(source_dir, "openstack", "latest"))):
-        LOG.warn("version '%s' not available, attempting to use 'latest'" %
-                 version)
-        version = "latest"
-
-    datafiles = (
-        ('metadata',
-         "openstack/%s/meta_data.json" % version, True, json.loads),
-        ('userdata', "openstack/%s/user_data" % version, False, None),
-        ('ec2-metadata', "ec2/latest/meta-data.json", False, json.loads),
-    )
-
-    results = {'userdata': None}
-    for (name, path, required, process) in datafiles:
-        fpath = os.path.join(source_dir, path)
-        data = None
-        found = False
-        if os.path.isfile(fpath):
-            try:
-                data = util.load_file(fpath)
-            except IOError:
-                raise BrokenConfigDriveDir("Failed to read: %s" % fpath)
-            found = True
-        elif required:
-            raise NonConfigDriveDir("Missing mandatory path: %s" % fpath)
-
-        if found and process:
-            try:
-                data = process(data)
-            except Exception as exc:
-                raise BrokenConfigDriveDir(("Failed to process "
-                                            "path: %s") % fpath)
-
-        if found:
-            results[name] = data
-
-    # instance-id is 'uuid' for openstack. just copy it to instance-id.
-    if 'instance-id' not in results['metadata']:
-        try:
-            results['metadata']['instance-id'] = results['metadata']['uuid']
-        except KeyError:
-            raise BrokenConfigDriveDir("No uuid entry in metadata")
-
-    if 'random_seed' in results['metadata']:
-        random_seed = results['metadata']['random_seed']
-        try:
-            results['metadata']['random_seed'] = base64.b64decode(random_seed)
-        except (ValueError, TypeError) as exc:
-            raise BrokenConfigDriveDir("Badly formatted random_seed: %s" % exc)
-
-    def read_content_path(item):
-        # do not use os.path.join here, as content_path starts with /
-        cpath = os.path.sep.join((source_dir, "openstack",
-                                  "./%s" % item['content_path']))
-        return util.load_file(cpath)
-
-    files = {}
-    try:
-        for item in results['metadata'].get('files', {}):
-            files[item['path']] = read_content_path(item)
-
-        # the 'network_config' item in metadata is a content pointer
-        # to the network config that should be applied.
-        # in folsom, it is just a '/etc/network/interfaces' file.
-        item = results['metadata'].get("network_config", None)
-        if item:
-            results['network_config'] = read_content_path(item)
-    except Exception as exc:
-        raise BrokenConfigDriveDir("Failed to read file %s: %s" % (item, exc))
-
-    # to openstack, user can specify meta ('nova boot --meta=key=value') and
-    # those will appear under metadata['meta'].
-    # if they specify 'dsmode' they're indicating the mode that they intend
-    # for this datasource to operate in.
-    try:
-        results['dsmode'] = results['metadata']['meta']['dsmode']
-    except KeyError:
-        pass
-
-    results['files'] = files
-    results['cfgdrive_ver'] = 2
-    return results
-
-
-def read_config_drive_dir_v1(source_dir):
-    """
-    read source_dir, and return a tuple with metadata dict, user-data,
-    files and version (1).  If not a valid dir, raise a NonConfigDriveDir
-    """
-
-    found = {}
-    for af in CFG_DRIVE_FILES_V1:
-        fn = os.path.join(source_dir, af)
-        if os.path.isfile(fn):
-            found[af] = fn
-
-    if len(found) == 0:
-        raise NonConfigDriveDir("%s: %s" % (source_dir, "no files found"))
-
-    md = {}
-    keydata = ""
-    if "etc/network/interfaces" in found:
-        fn = found["etc/network/interfaces"]
-        md['network_config'] = util.load_file(fn)
-
-    if "root/.ssh/authorized_keys" in found:
-        fn = found["root/.ssh/authorized_keys"]
-        keydata = util.load_file(fn)
-
-    meta_js = {}
-    if "meta.js" in found:
-        fn = found['meta.js']
-        content = util.load_file(fn)
-        try:
-            # Just check if its really json...
-            meta_js = json.loads(content)
-            if not isinstance(meta_js, (dict)):
-                raise TypeError("Dict expected for meta.js root node")
-        except (ValueError, TypeError) as e:
-            raise NonConfigDriveDir("%s: %s, %s" %
-                (source_dir, "invalid json in meta.js", e))
-        md['meta_js'] = content
-
-    # keydata in meta_js is preferred over "injected"
-    keydata = meta_js.get('public-keys', keydata)
-    if keydata:
-        lines = keydata.splitlines()
-        md['public-keys'] = [l for l in lines
-            if len(l) and not l.startswith("#")]
-
-    # config-drive-v1 has no way for openstack to provide the instance-id
-    # so we copy that into metadata from the user input
-    if 'instance-id' in meta_js:
-        md['instance-id'] = meta_js['instance-id']
-
-    results = {'cfgdrive_ver': 1, 'metadata': md}
-
-    # allow the user to specify 'dsmode' in a meta tag
-    if 'dsmode' in meta_js:
-        results['dsmode'] = meta_js['dsmode']
-
-    # config-drive-v1 has no way of specifying user-data, so the user has
-    # to cheat and stuff it in a meta tag also.
-    results['userdata'] = meta_js.get('user-data')
-
-    # this implementation does not support files
-    # (other than network/interfaces and authorized_keys)
-    results['files'] = []
-
-    return results
-
-
 def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None):
     """Determine what mode should be used.
     valid values are 'pass', 'disabled', 'local', 'net'
@@ -483,6 +160,21 @@ def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None):
     return "net"
 
 
+def read_config_drive(source_dir, version="2012-08-10"):
+    reader = openstack.ConfigDriveReader(source_dir)
+    finders = [
+        (reader.read_v2, [], {'version': version}),
+        (reader.read_v1, [], {}),
+    ]
+    excps = []
+    for (functor, args, kwargs) in finders:
+        try:
+            return functor(*args, **kwargs)
+        except openstack.NonReadable as e:
+            excps.append(e)
+    raise excps[-1]
+
+
 def get_previous_iid(paths):
     # interestingly, for this purpose the "previous" instance-id is the current
     # instance-id.  cloud-init hasn't moved them over yet as this datasource
@@ -494,17 +186,79 @@ def get_previous_iid(paths):
         return None
 
 
-def write_files(files):
-    for (name, content) in files.iteritems():
-        if name[0] != os.sep:
-            name = os.sep + name
-        util.write_file(name, content, mode=0660)
+def on_first_boot(data, distro=None):
+    """Performs any first-boot actions using data read from a config-drive."""
+    if not isinstance(data, dict):
+        raise TypeError("Config-drive data expected to be a dict; not %s"
+                        % (type(data)))
+    net_conf = data.get("network_config", '')
+    if net_conf and distro:
+        LOG.debug("Updating network interfaces from config drive")
+        distro.apply_network(net_conf)
+    files = data.get('files', {})
+    if files:
+        LOG.debug("Writing %s injected files", len(files))
+        for (filename, content) in files.iteritems():
+            if not filename.startswith(os.sep):
+                filename = os.sep + filename
+            try:
+                util.write_file(filename, content, mode=0660)
+            except IOError:
+                util.logexc(LOG, "Failed writing file: %s", filename)
+
+
+def find_candidate_devs(probe_optical=True):
+    """Return a list of devices that may contain the config drive.
+
+    The returned list is sorted by search order where the first item has
+    should be searched first (highest priority)
+
+    config drive v1:
+       Per documentation, this is "associated as the last available disk on the
+       instance", and should be VFAT.
+       Currently, we do not restrict search list to "last available disk"
+
+    config drive v2:
+       Disk should be:
+        * either vfat or iso9660 formated
+        * labeled with 'config-2'
+    """
+    # query optical drive to get it in blkid cache for 2.6 kernels
+    if probe_optical:
+        for device in OPTICAL_DEVICES:
+            try:
+                util.find_devs_with(path=device)
+            except util.ProcessExecutionError:
+                pass
+
+    by_fstype = []
+    for fs_type in FS_TYPES:
+        by_fstype.extend(util.find_devs_with("TYPE=%s" % (fs_type)))
+
+    by_label = []
+    for label in LABEL_TYPES:
+        by_label.extend(util.find_devs_with("LABEL=%s" % (label)))
+
+    # give preference to "last available disk" (vdb over vda)
+    # note, this is not a perfect rendition of that.
+    by_fstype.sort(reverse=True)
+    by_label.sort(reverse=True)
+
+    # combine list of items by putting by-label items first
+    # followed by fstype items, but with dupes removed
+    candidates = (by_label + [d for d in by_fstype if d not in by_label])
+
+    # We are looking for a block device or partition with necessary label or
+    # an unpartitioned block device (ex sda, not sda1)
+    devices = [d for d in candidates
+               if d in by_label or not util.is_partition(d)]
+    return devices
 
 
 # Used to match classes to dependencies
 datasources = [
-  (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )),
-  (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+    (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )),
+    (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
 ]
 
 
diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py
new file mode 100644
index 00000000..c993293f
--- /dev/null
+++ b/cloudinit/sources/DataSourceGCE.py
@@ -0,0 +1,138 @@
+# vi: ts=4 expandtab
+#
+#    Author: Vaidas Jablonskis <jablonskis@gmail.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from cloudinit import log as logging
+from cloudinit import util
+from cloudinit import sources
+from cloudinit import url_helper
+
+LOG = logging.getLogger(__name__)
+
+BUILTIN_DS_CONFIG = {
+    'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/'
+}
+REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname')
+
+
+class DataSourceGCE(sources.DataSource):
+    def __init__(self, sys_cfg, distro, paths):
+        sources.DataSource.__init__(self, sys_cfg, distro, paths)
+        self.metadata = dict()
+        self.ds_cfg = util.mergemanydict([
+            util.get_cfg_by_path(sys_cfg, ["datasource", "GCE"], {}),
+            BUILTIN_DS_CONFIG])
+        self.metadata_address = self.ds_cfg['metadata_url']
+
+    # GCE takes sshKeys attribute in the format of '<user>:<public_key>'
+    # so we have to trim each key to remove the username part
+    def _trim_key(self, public_key):
+        try:
+            index = public_key.index(':')
+            if index > 0:
+                return public_key[(index + 1):]
+        except:
+            return public_key
+
+    def get_data(self):
+        # GCE metadata server requires a custom header since v1
+        headers = {'X-Google-Metadata-Request': True}
+
+        # url_map: (our-key, path, required)
+        url_map = [
+            ('instance-id', 'instance/id', True),
+            ('availability-zone', 'instance/zone', True),
+            ('local-hostname', 'instance/hostname', True),
+            ('public-keys', 'project/attributes/sshKeys', False),
+            ('user-data', 'instance/attributes/user-data', False),
+        ]
+
+        # if we cannot resolve the metadata server, then no point in trying
+        if not util.is_resolvable(self.metadata_address):
+            LOG.debug("%s is not resolvable", self.metadata_address)
+            return False
+
+        # iterate over url_map keys to get metadata items
+        found = False
+        for (mkey, path, required) in url_map:
+            try:
+                resp = url_helper.readurl(url=self.metadata_address + path,
+                                          headers=headers)
+                if resp.code == 200:
+                    found = True
+                    self.metadata[mkey] = resp.contents
+                else:
+                    if required:
+                        msg = "required url %s returned code %s. not GCE"
+                        if not found:
+                            LOG.debug(msg, path, resp.code)
+                        else:
+                            LOG.warn(msg, path, resp.code)
+                        return False
+                    else:
+                        self.metadata[mkey] = None
+            except url_helper.UrlError as e:
+                if required:
+                    msg = "required url %s raised exception %s. not GCE"
+                    if not found:
+                        LOG.debug(msg, path, e)
+                    else:
+                        LOG.warn(msg, path, e)
+                    return False
+                msg = "Failed to get %s metadata item: %s."
+                if found:
+                    LOG.warn(msg, path, e)
+                else:
+                    LOG.debug(msg, path, e)
+
+                self.metadata[mkey] = None
+
+        if self.metadata['public-keys']:
+            lines = self.metadata['public-keys'].splitlines()
+            self.metadata['public-keys'] = [self._trim_key(k) for k in lines]
+
+        return found
+
+    @property
+    def launch_index(self):
+        # GCE does not provide lauch_index property
+        return None
+
+    def get_instance_id(self):
+        return self.metadata['instance-id']
+
+    def get_public_ssh_keys(self):
+        return self.metadata['public-keys']
+
+    def get_hostname(self, fqdn=False, _resolve_ip=False):
+        return self.metadata['local-hostname']
+
+    def get_userdata_raw(self):
+        return self.metadata['user-data']
+
+    @property
+    def availability_zone(self):
+        return self.metadata['availability-zone']
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
new file mode 100644
index 00000000..5edbb448
--- /dev/null
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -0,0 +1,162 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2014 Yahoo! Inc.
+#
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import time
+
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import url_helper
+from cloudinit import util
+
+from cloudinit.sources.helpers import openstack
+
+LOG = logging.getLogger(__name__)
+
+# Various defaults/constants...
+DEF_MD_URL = "http://169.254.169.254"
+DEFAULT_IID = "iid-dsopenstack"
+DEFAULT_METADATA = {
+    "instance-id": DEFAULT_IID,
+}
+VALID_DSMODES = ("net", "disabled")
+
+
+class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
+    def __init__(self, sys_cfg, distro, paths):
+        super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths)
+        self.dsmode = 'net'
+        self.metadata_address = None
+        self.ssl_details = util.fetch_ssl_details(self.paths)
+        self.version = None
+        self.files = {}
+        self.ec2_metadata = None
+        if not self.ds_cfg:
+            self.ds_cfg = {}
+
+    def __str__(self):
+        root = sources.DataSource.__str__(self)
+        mstr = "%s [%s,ver=%s]" % (root, self.dsmode, self.version)
+        return mstr
+
+    def _get_url_settings(self):
+        # TODO(harlowja): this is shared with ec2 datasource, we should just
+        # move it to a shared location instead...
+        # Note: the defaults here are different though.
+
+        # max_wait < 0 indicates do not wait
+        max_wait = -1
+        timeout = 10
+
+        try:
+            max_wait = int(self.ds_cfg.get("max_wait", max_wait))
+        except Exception:
+            util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
+
+        try:
+            timeout = max(0, int(self.ds_cfg.get("timeout", timeout)))
+        except Exception:
+            util.logexc(LOG, "Failed to get timeout, using %s", timeout)
+        return (max_wait, timeout)
+
+    def wait_for_metadata_service(self):
+        urls = self.ds_cfg.get("metadata_urls", [DEF_MD_URL])
+        filtered = [x for x in urls if util.is_resolvable_url(x)]
+        if set(filtered) != set(urls):
+            LOG.debug("Removed the following from metadata urls: %s",
+                      list((set(urls) - set(filtered))))
+        if len(filtered):
+            urls = filtered
+        else:
+            LOG.warn("Empty metadata url list! using default list")
+            urls = [DEF_MD_URL]
+
+        md_urls = []
+        url2base = {}
+        for url in urls:
+            md_url = url_helper.combine_url(url, 'openstack',
+                                            openstack.OS_LATEST,
+                                            'meta_data.json')
+            md_urls.append(md_url)
+            url2base[md_url] = url
+
+        (max_wait, timeout) = self._get_url_settings()
+        start_time = time.time()
+        avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait,
+                                            timeout=timeout)
+        if avail_url:
+            LOG.debug("Using metadata source: '%s'", url2base[avail_url])
+        else:
+            LOG.debug("Giving up on OpenStack md from %s after %s seconds",
+                      md_urls, int(time.time() - start_time))
+
+        self.metadata_address = url2base.get(avail_url)
+        return bool(avail_url)
+
+    def get_data(self):
+        try:
+            if not self.wait_for_metadata_service():
+                return False
+        except IOError:
+            return False
+
+        try:
+            results = util.log_time(LOG.debug,
+                                    'Crawl of openstack metadata service',
+                                    read_metadata_service,
+                                    args=[self.metadata_address],
+                                    kwargs={'ssl_details': self.ssl_details,
+                                            'version': openstack.OS_LATEST})
+        except openstack.NonReadable:
+            return False
+        except (openstack.BrokenMetadata, IOError):
+            util.logexc(LOG, "Broken metadata address %s",
+                        self.metadata_address)
+            return False
+
+        user_dsmode = results.get('dsmode', None)
+        if user_dsmode not in VALID_DSMODES + (None,):
+            LOG.warn("User specified invalid mode: %s", user_dsmode)
+            user_dsmode = None
+        if user_dsmode == 'disabled':
+            return False
+
+        md = results.get('metadata', {})
+        md = util.mergemanydict([md, DEFAULT_METADATA])
+        self.metadata = md
+        self.ec2_metadata = results.get('ec2-metadata')
+        self.userdata_raw = results.get('userdata')
+        self.version = results['version']
+        self.files.update(results.get('files', {}))
+        self.vendordata_raw = results.get('vendordata')
+        return True
+
+
+def read_metadata_service(base_url, version=None, ssl_details=None):
+    reader = openstack.MetadataReader(base_url, ssl_details=ssl_details)
+    return reader.read_v2(version=version)
+
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceOpenStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
diff --git a/cloudinit/sources/helpers/__init__.py b/cloudinit/sources/helpers/__init__.py
new file mode 100644
index 00000000..2cf99ec8
--- /dev/null
+++ b/cloudinit/sources/helpers/__init__.py
@@ -0,0 +1,14 @@
+# vi: ts=4 expandtab
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
new file mode 100644
index 00000000..a17148d3
--- /dev/null
+++ b/cloudinit/sources/helpers/openstack.py
@@ -0,0 +1,436 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Canonical Ltd.
+#    Copyright (C) 2012 Yahoo! Inc.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import abc
+import base64
+import copy
+import os
+
+from cloudinit import ec2_utils
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import url_helper
+from cloudinit import util
+
+# For reference: http://tinyurl.com/laora4c
+
+LOG = logging.getLogger(__name__)
+
+FILES_V1 = {
+    # Path <-> (metadata key name, translator function, default value)
+    'etc/network/interfaces': ('network_config', lambda x: x, ''),
+    'meta.js': ('meta_js', util.load_json, {}),
+    "root/.ssh/authorized_keys": ('authorized_keys', lambda x: x, ''),
+}
+KEY_COPIES = (
+    # Cloud-init metadata names <-> (metadata key, is required)
+    ('local-hostname', 'hostname', False),
+    ('instance-id', 'uuid', True),
+)
+OS_VERSIONS = (
+    '2012-08-10',  # folsom
+    '2013-04-04',  # grizzly
+    '2013-10-17',  # havana
+)
+OS_LATEST = 'latest'
+
+
+class NonReadable(IOError):
+    pass
+
+
+class BrokenMetadata(IOError):
+    pass
+
+
+class SourceMixin(object):
+    def _ec2_name_to_device(self, name):
+        if not self.ec2_metadata:
+            return None
+        bdm = self.ec2_metadata.get('block-device-mapping', {})
+        for (ent_name, device) in bdm.items():
+            if name == ent_name:
+                return device
+        return None
+
+    def get_public_ssh_keys(self):
+        name = "public_keys"
+        if self.version == 1:
+            name = "public-keys"
+        return sources.normalize_pubkey_data(self.metadata.get(name))
+
+    def _os_name_to_device(self, name):
+        device = None
+        try:
+            criteria = 'LABEL=%s' % (name)
+            if name == 'swap':
+                criteria = 'TYPE=%s' % (name)
+            dev_entries = util.find_devs_with(criteria)
+            if dev_entries:
+                device = dev_entries[0]
+        except util.ProcessExecutionError:
+            pass
+        return device
+
+    def _validate_device_name(self, device):
+        if not device:
+            return None
+        if not device.startswith("/"):
+            device = "/dev/%s" % device
+        if os.path.exists(device):
+            return device
+        # Durn, try adjusting the mapping
+        remapped = self._remap_device(os.path.basename(device))
+        if remapped:
+            LOG.debug("Remapped device name %s => %s", device, remapped)
+            return remapped
+        return None
+
+    def device_name_to_device(self, name):
+        # Translate a 'name' to a 'physical' device
+        if not name:
+            return None
+        # Try the ec2 mapping first
+        names = [name]
+        if name == 'root':
+            names.insert(0, 'ami')
+        if name == 'ami':
+            names.append('root')
+        device = None
+        LOG.debug("Using ec2 style lookup to find device %s", names)
+        for n in names:
+            device = self._ec2_name_to_device(n)
+            device = self._validate_device_name(device)
+            if device:
+                break
+        # Try the openstack way second
+        if not device:
+            LOG.debug("Using openstack style lookup to find device %s", names)
+            for n in names:
+                device = self._os_name_to_device(n)
+                device = self._validate_device_name(device)
+                if device:
+                    break
+        # Ok give up...
+        if not device:
+            return None
+        else:
+            LOG.debug("Mapped %s to device %s", name, device)
+            return device
+
+
+class BaseReader(object):
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, base_path):
+        self.base_path = base_path
+
+    @abc.abstractmethod
+    def _path_join(self, base, *add_ons):
+        pass
+
+    @abc.abstractmethod
+    def _path_exists(self, path):
+        pass
+
+    @abc.abstractmethod
+    def _path_read(self, path):
+        pass
+
+    @abc.abstractmethod
+    def _read_ec2_metadata(self):
+        pass
+
+    def _read_content_path(self, item):
+        path = item.get('content_path', '').lstrip("/")
+        path_pieces = path.split("/")
+        valid_pieces = [p for p in path_pieces if len(p)]
+        if not valid_pieces:
+            raise BrokenMetadata("Item %s has no valid content path" % (item))
+        path = self._path_join(self.base_path, "openstack", *path_pieces)
+        return self._path_read(path)
+
+    def _find_working_version(self, version):
+        search_versions = [version] + list(OS_VERSIONS)
+        for potential_version in search_versions:
+            if not potential_version:
+                continue
+            path = self._path_join(self.base_path, "openstack",
+                                   potential_version)
+            if self._path_exists(path):
+                if potential_version != version:
+                    LOG.warn("Version '%s' not available, attempting to use"
+                             " version '%s' instead", version,
+                             potential_version)
+                return potential_version
+        LOG.warn("Version '%s' not available, attempting to use '%s'"
+                 " instead", version, OS_LATEST)
+        return OS_LATEST
+
+    def read_v2(self, version=None):
+        """Reads a version 2 formatted location.
+
+        Return a dict with metadata, userdata, ec2-metadata, dsmode,
+        network_config, files and version (2).
+
+        If not a valid location, raise a NonReadable exception.
+        """
+
+        def datafiles(version):
+            files = {}
+            files['metadata'] = (
+                # File path to read
+                self._path_join("openstack", version, 'meta_data.json'),
+                # Is it required?
+                True,
+                # Translator function (applied after loading)
+                util.load_json,
+            )
+            files['userdata'] = (
+                self._path_join("openstack", version, 'user_data'),
+                False,
+                lambda x: x,
+            )
+            files['vendordata'] = (
+                self._path_join("openstack", version, 'vendor_data.json'),
+                False,
+                util.load_json,
+            )
+            return files
+
+        version = self._find_working_version(version)
+        results = {
+            'userdata': '',
+            'version': 2,
+        }
+        data = datafiles(version)
+        for (name, (path, required, translator)) in data.iteritems():
+            path = self._path_join(self.base_path, path)
+            data = None
+            found = False
+            if self._path_exists(path):
+                try:
+                    data = self._path_read(path)
+                except IOError:
+                    raise NonReadable("Failed to read: %s" % path)
+                found = True
+            else:
+                if required:
+                    raise NonReadable("Missing mandatory path: %s" % path)
+            if found and translator:
+                try:
+                    data = translator(data)
+                except Exception as e:
+                    raise BrokenMetadata("Failed to process "
+                                         "path %s: %s" % (path, e))
+            if found:
+                results[name] = data
+
+        metadata = results['metadata']
+        if 'random_seed' in metadata:
+            random_seed = metadata['random_seed']
+            try:
+                metadata['random_seed'] = base64.b64decode(random_seed)
+            except (ValueError, TypeError) as e:
+                raise BrokenMetadata("Badly formatted metadata"
+                                     " random_seed entry: %s" % e)
+
+        # load any files that were provided
+        files = {}
+        metadata_files = metadata.get('files', [])
+        for item in metadata_files:
+            if 'path' not in item:
+                continue
+            path = item['path']
+            try:
+                files[path] = self._read_content_path(item)
+            except Exception as e:
+                raise BrokenMetadata("Failed to read provided "
+                                     "file %s: %s" % (path, e))
+        results['files'] = files
+
+        # The 'network_config' item in metadata is a content pointer
+        # to the network config that should be applied. It is just a
+        # ubuntu/debian '/etc/network/interfaces' file.
+        net_item = metadata.get("network_config", None)
+        if net_item:
+            try:
+                results['network_config'] = self._read_content_path(net_item)
+            except IOError as e:
+                raise BrokenMetadata("Failed to read network"
+                                     " configuration: %s" % (e))
+
+        # To openstack, user can specify meta ('nova boot --meta=key=value')
+        # and those will appear under metadata['meta'].
+        # if they specify 'dsmode' they're indicating the mode that they intend
+        # for this datasource to operate in.
+        try:
+            results['dsmode'] = metadata['meta']['dsmode']
+        except KeyError:
+            pass
+
+        # Read any ec2-metadata (if applicable)
+        results['ec2-metadata'] = self._read_ec2_metadata()
+
+        # Perform some misc. metadata key renames...
+        for (target_key, source_key, is_required) in KEY_COPIES:
+            if is_required and source_key not in metadata:
+                raise BrokenMetadata("No '%s' entry in metadata" % source_key)
+            if source_key in metadata:
+                metadata[target_key] = metadata.get(source_key)
+        return results
+
+
+class ConfigDriveReader(BaseReader):
+    def __init__(self, base_path):
+        super(ConfigDriveReader, self).__init__(base_path)
+
+    def _path_join(self, base, *add_ons):
+        components = [base] + list(add_ons)
+        return os.path.join(*components)
+
+    def _path_exists(self, path):
+        return os.path.exists(path)
+
+    def _path_read(self, path):
+        return util.load_file(path)
+
+    def _read_ec2_metadata(self):
+        path = self._path_join(self.base_path,
+                               'ec2', 'latest', 'meta-data.json')
+        if not self._path_exists(path):
+            return {}
+        else:
+            try:
+                return util.load_json(self._path_read(path))
+            except Exception as e:
+                raise BrokenMetadata("Failed to process "
+                                     "path %s: %s" % (path, e))
+
+    def read_v1(self):
+        """Reads a version 1 formatted location.
+
+        Return a dict with metadata, userdata, dsmode, files and version (1).
+
+        If not a valid path, raise a NonReadable exception.
+        """
+
+        found = {}
+        for name in FILES_V1.keys():
+            path = self._path_join(self.base_path, name)
+            if self._path_exists(path):
+                found[name] = path
+        if len(found) == 0:
+            raise NonReadable("%s: no files found" % (self.base_path))
+
+        md = {}
+        for (name, (key, translator, default)) in FILES_V1.iteritems():
+            if name in found:
+                path = found[name]
+                try:
+                    contents = self._path_read(path)
+                except IOError:
+                    raise BrokenMetadata("Failed to read: %s" % path)
+                try:
+                    md[key] = translator(contents)
+                except Exception as e:
+                    raise BrokenMetadata("Failed to process "
+                                         "path %s: %s" % (path, e))
+            else:
+                md[key] = copy.deepcopy(default)
+
+        keydata = md['authorized_keys']
+        meta_js = md['meta_js']
+
+        # keydata in meta_js is preferred over "injected"
+        keydata = meta_js.get('public-keys', keydata)
+        if keydata:
+            lines = keydata.splitlines()
+            md['public-keys'] = [l for l in lines
+                                 if len(l) and not l.startswith("#")]
+
+        # config-drive-v1 has no way for openstack to provide the instance-id
+        # so we copy that into metadata from the user input
+        if 'instance-id' in meta_js:
+            md['instance-id'] = meta_js['instance-id']
+
+        results = {
+            'version': 1,
+            'metadata': md,
+        }
+
+        # allow the user to specify 'dsmode' in a meta tag
+        if 'dsmode' in meta_js:
+            results['dsmode'] = meta_js['dsmode']
+
+        # config-drive-v1 has no way of specifying user-data, so the user has
+        # to cheat and stuff it in a meta tag also.
+        results['userdata'] = meta_js.get('user-data', '')
+
+        # this implementation does not support files other than
+        # network/interfaces and authorized_keys...
+        results['files'] = {}
+
+        return results
+
+
+class MetadataReader(BaseReader):
+    def __init__(self, base_url, ssl_details=None, timeout=5, retries=5):
+        super(MetadataReader, self).__init__(base_url)
+        self.ssl_details = ssl_details
+        self.timeout = float(timeout)
+        self.retries = int(retries)
+
+    def _path_read(self, path):
+        response = url_helper.readurl(path,
+                                      retries=self.retries,
+                                      ssl_details=self.ssl_details,
+                                      timeout=self.timeout)
+        return response.contents
+
+    def _path_exists(self, path):
+
+        def should_retry_cb(request, cause):
+            try:
+                code = int(cause.code)
+                if code >= 400:
+                    return False
+            except (TypeError, ValueError):
+                # Older versions of requests didn't have a code.
+                pass
+            return True
+
+        try:
+            response = url_helper.readurl(path,
+                                          retries=self.retries,
+                                          ssl_details=self.ssl_details,
+                                          timeout=self.timeout,
+                                          exception_cb=should_retry_cb)
+            return response.ok()
+        except IOError:
+            return False
+
+    def _path_join(self, base, *add_ons):
+        return url_helper.combine_url(base, *add_ons)
+
+    def _read_ec2_metadata(self):
+        return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details,
+                                               timeout=self.timeout,
+                                               retries=self.retries)
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 7acd3355..58349ffc 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -234,7 +234,7 @@ class Init(object):
                                                copy.deepcopy(self.ds_deps),
                                                cfg_list,
                                                pkg_list)
-            LOG.debug("Loaded datasource %s - %s", dsname, ds)
+            LOG.info("Loaded datasource %s - %s", dsname, ds)
         self.datasource = ds
         # Ensure we adjust our path members datasource
         # now that we have one (thus allowing ipath to be used)
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 97ed75ad..4a83169a 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -22,6 +22,7 @@
 
 import httplib
 import time
+import urllib
 
 import requests
 from requests import exceptions
@@ -38,6 +39,7 @@ NOT_FOUND = httplib.NOT_FOUND
 # Check if requests has ssl support (added in requests >= 0.8.8)
 SSL_ENABLED = False
 CONFIG_ENABLED = False  # This was added in 0.7 (but taken out in >=1.0)
+_REQ_VER = None
 try:
     from distutils.version import LooseVersion
     import pkg_resources
@@ -61,6 +63,23 @@ def _cleanurl(url):
     return urlunparse(parsed_url)
 
 
+def combine_url(base, *add_ons):
+
+    def combine_single(url, add_on):
+        url_parsed = list(urlparse(url))
+        path = url_parsed[2]
+        if path and not path.endswith("/"):
+            path += "/"
+        path += urllib.quote(str(add_on), safe="/:")
+        url_parsed[2] = path
+        return urlunparse(url_parsed)
+
+    url = base
+    for add_on in add_ons:
+        url = combine_single(url, add_on)
+    return url
+
+
 # Made to have same accessors as UrlResponse so that the
 # read_file_or_url can return this or that object and the
 # 'user' of those objects will not need to know the difference.
@@ -129,28 +148,34 @@ class UrlError(IOError):
             self.headers = {}
 
 
-def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
-            headers=None, headers_cb=None, ssl_details=None,
-            check_status=True, allow_redirects=True, exception_cb=None):
-    url = _cleanurl(url)
-    req_args = {
-        'url': url,
-    }
+def _get_ssl_args(url, ssl_details):
+    ssl_args = {}
     scheme = urlparse(url).scheme  # pylint: disable=E1101
     if scheme == 'https' and ssl_details:
         if not SSL_ENABLED:
-            LOG.warn("SSL is not enabled, cert. verification can not occur!")
+            LOG.warn("SSL is not supported in requests v%s, "
+                     "cert. verification can not occur!", _REQ_VER)
         else:
             if 'ca_certs' in ssl_details and ssl_details['ca_certs']:
-                req_args['verify'] = ssl_details['ca_certs']
+                ssl_args['verify'] = ssl_details['ca_certs']
             else:
-                req_args['verify'] = True
+                ssl_args['verify'] = True
             if 'cert_file' in ssl_details and 'key_file' in ssl_details:
-                req_args['cert'] = [ssl_details['cert_file'],
+                ssl_args['cert'] = [ssl_details['cert_file'],
                                     ssl_details['key_file']]
             elif 'cert_file' in ssl_details:
-                req_args['cert'] = str(ssl_details['cert_file'])
+                ssl_args['cert'] = str(ssl_details['cert_file'])
+    return ssl_args
 
+
+def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
+            headers=None, headers_cb=None, ssl_details=None,
+            check_status=True, allow_redirects=True, exception_cb=None):
+    url = _cleanurl(url)
+    req_args = {
+        'url': url,
+    }
+    req_args.update(_get_ssl_args(url, ssl_details))
     req_args['allow_redirects'] = allow_redirects
     req_args['method'] = 'GET'
     if timeout is not None:
@@ -181,12 +206,11 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
         def _cb(url):
             return headers
         headers_cb = _cb
-
     if data:
-        # Do this after the log (it might be large)
         req_args['data'] = data
     if sec_between is None:
         sec_between = -1
+
     excps = []
     # Handle retrying ourselves since the built-in support
     # doesn't handle sleeping between tries...
@@ -223,7 +247,7 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
                     # ssl exceptions are not going to get fixed by waiting a
                     # few seconds
                     break
-            if exception_cb and not exception_cb(filtered_req_args, excps[-1]):
+            if exception_cb and not exception_cb(req_args.copy(), excps[-1]):
                 break
             if i + 1 < manual_tries and sec_between > 0:
                 LOG.debug("Please wait %s seconds while we wait to try again",
@@ -242,6 +266,7 @@ def wait_for_url(urls, max_wait=None, timeout=None,
     max_wait:  roughly the maximum time to wait before giving up
                The max time is *actually* len(urls)*timeout as each url will
                be tried once and given the timeout provided.
+               a number <= 0 will always result in only one try
     timeout:   the timeout provided to urlopen
     status_cb: call method with string message when a url is not available
     headers_cb: call method with single argument of url to get headers
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 08cdd8c8..87b0c853 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -32,6 +32,7 @@ import glob
 import grp
 import gzip
 import hashlib
+import json
 import os
 import os.path
 import platform
@@ -362,6 +363,15 @@ def multi_log(text, console=True, stderr=True,
             log.log(log_level, text)
 
 
+def load_json(text, root_types=(dict,)):
+    decoded = json.loads(text)
+    if not isinstance(decoded, tuple(root_types)):
+        expected_types = ", ".join([str(t) for t in root_types])
+        raise TypeError("(%s) root types expected, got %s instead"
+                        % (expected_types, type(decoded)))
+    return decoded
+
+
 def is_ipv4(instr):
     """determine if input string is a ipv4 address. return boolean."""
     toks = instr.split('.')