Don't forget the rest of the files!

3 files changed, 597 insertions, 0 deletions

diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
new file mode 100644
index 00000000..44889f4e
--- /dev/null
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -0,0 +1,163 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2014 Yahoo! Inc.
+#
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import time
+
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import url_helper
+from cloudinit import util
+
+from cloudinit.sources.helpers import openstack
+
+LOG = logging.getLogger(__name__)
+
+# Various defaults/constants...
+DEF_MD_URL = "http://169.254.169.254"
+DEFAULT_IID = "iid-dsopenstack"
+DEFAULT_METADATA = {
+    "instance-id": DEFAULT_IID,
+}
+VALID_DSMODES = ("net", "disabled")
+
+
+class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
+    def __init__(self, sys_cfg, distro, paths):
+        super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths)
+        self.dsmode = 'net'
+        self.metadata_address = None
+        self.ssl_details = util.fetch_ssl_details(self.paths)
+        self.version = None
+        self.files = {}
+
+    def __str__(self):
+        root = sources.DataSource.__str__(self)
+        mstr = "%s [%s,ver=%s]" % (root, self.dsmode, self.version)
+        return mstr
+
+    def _get_url_settings(self):
+        # TODO(harlowja): this is shared with ec2 datasource, we should just
+        # move it to a shared location instead...
+        ds_cfg = self.ds_cfg
+        if not ds_cfg:
+            ds_cfg = {}
+        max_wait = 120
+        try:
+            max_wait = int(ds_cfg.get("max_wait", max_wait))
+        except Exception:
+            util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
+
+        timeout = 50
+        try:
+            timeout = max(0, int(ds_cfg.get("timeout", timeout)))
+        except Exception:
+            util.logexc(LOG, "Failed to get timeout, using %s", timeout)
+        return (max_wait, timeout)
+
+    def wait_for_metadata_service(self):
+        ds_cfg = self.ds_cfg
+        if not ds_cfg:
+            ds_cfg = {}
+        urls = ds_cfg.get("metadata_urls", [DEF_MD_URL])
+        filtered = [x for x in urls if util.is_resolvable_url(x)]
+        if set(filtered) != set(urls):
+            LOG.debug("Removed the following from metadata urls: %s",
+                      list((set(urls) - set(filtered))))
+        if len(filtered):
+            urls = filtered
+        else:
+            LOG.warn("Empty metadata url list! using default list")
+            urls = [DEF_MD_URL]
+
+        md_urls = []
+        url2base = {}
+        for url in urls:
+            md_url = url_helper.combine_url(url, 'openstack',
+                                            openstack.OS_LATEST,
+                                            'meta_data.json')
+            md_urls.append(md_url)
+            url2base[md_url] = url
+
+        (max_wait, timeout) = self._get_url_settings()
+        if max_wait <= 0:
+            return False
+        start_time = time.time()
+        avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait,
+                                            timeout=timeout,
+                                            status_cb=LOG.warn)
+        if avail_url:
+            LOG.debug("Using metadata source: '%s'", url2base[avail_url])
+        else:
+            LOG.critical("Giving up on md from %s after %s seconds",
+                         md_urls, int(time.time() - start_time))
+
+        self.metadata_address = url2base.get(avail_url)
+        return bool(avail_url)
+
+    def get_data(self):
+        try:
+            if not self.wait_for_metadata_service():
+                return False
+        except IOError:
+            return False
+
+        try:
+            results = util.log_time(LOG.debug,
+                                    'Crawl of openstack metadata service',
+                                    read_metadata_service,
+                                    args=[self.metadata_address],
+                                    kwargs={'ssl_details': self.ssl_details})
+        except openstack.NonReadable:
+            return False
+        except openstack.BrokenMetadata:
+            util.logexc(LOG, "Broken metadata address %s",
+                        self.metadata_address)
+            return False
+
+        user_dsmode = results.get('dsmode', None)
+        if user_dsmode not in VALID_DSMODES + (None,):
+            LOG.warn("User specified invalid mode: %s" % user_dsmode)
+            user_dsmode = None
+        if user_dsmode == 'disabled':
+            return False
+
+        md = results.get('metadata', {})
+        md = util.mergemanydict([md, DEFAULT_METADATA])
+        self.metadata = md
+        self.ec2_metadata = results.get('ec2-metadata')
+        self.userdata_raw = results.get('userdata')
+        self.version = results['version']
+        self.files.update(results.get('files', {}))
+        self.vendordata_raw = results.get('vendordata')
+        return True
+
+
+def read_metadata_service(base_url, version=None, ssl_details=None):
+    reader = openstack.MetadataReader(base_url, ssl_details=ssl_details)
+    return reader.read_v2(version=version)
+
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceOpenStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
diff --git a/cloudinit/sources/helpers/__init__.py b/cloudinit/sources/helpers/__init__.py
new file mode 100644
index 00000000..2cf99ec8
--- /dev/null
+++ b/cloudinit/sources/helpers/__init__.py
@@ -0,0 +1,14 @@
+# vi: ts=4 expandtab
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
new file mode 100644
index 00000000..9dbef677
--- /dev/null
+++ b/cloudinit/sources/helpers/openstack.py
@@ -0,0 +1,420 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Canonical Ltd.
+#    Copyright (C) 2012 Yahoo! Inc.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import abc
+import base64
+import copy
+import functools
+import os
+
+from cloudinit import ec2_utils
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import url_helper
+from cloudinit import util
+
+# For reference: http://tinyurl.com/laora4c
+
+LOG = logging.getLogger(__name__)
+
+FILES_V1 = {
+    # Path <-> (metadata key name, translator function, default value)
+    'etc/network/interfaces': ('network_config', lambda x: x, ''),
+    'meta.js': ('meta_js', util.load_json, {}),
+    "root/.ssh/authorized_keys": ('authorized_keys', lambda x: x, ''),
+}
+KEY_COPIES = (
+    # Cloud-init metadata names <-> (metadata key, is required)
+    ('local-hostname', 'hostname', False),
+    ('instance-id', 'uuid', True),
+)
+OS_VERSIONS = (
+    '2012-08-10',  # folsom
+    '2013-04-04',  # grizzly
+    '2013-10-17',  # havana
+)
+OS_LATEST = 'latest'
+
+
+class NonReadable(IOError):
+    pass
+
+
+class BrokenMetadata(IOError):
+    pass
+
+
+class SourceMixin(object):
+    def _ec2_name_to_device(self, name):
+        if not self.ec2_metadata:
+            return None
+        bdm = self.ec2_metadata.get('block-device-mapping', {})
+        for (ent_name, device) in bdm.items():
+            if name == ent_name:
+                return device
+        return None
+
+    def get_public_ssh_keys(self):
+        name = "public_keys"
+        if self.version == 1:
+            name = "public-keys"
+        return sources.normalize_pubkey_data(self.metadata.get(name))
+
+    def _os_name_to_device(self, name):
+        device = None
+        try:
+            criteria = 'LABEL=%s' % (name)
+            if name == 'swap':
+                criteria = 'TYPE=%s' % (name)
+            dev_entries = util.find_devs_with(criteria)
+            if dev_entries:
+                device = dev_entries[0]
+        except util.ProcessExecutionError:
+            pass
+        return device
+
+    def _validate_device_name(self, device):
+        if not device:
+            return None
+        if not device.startswith("/"):
+            device = "/dev/%s" % device
+        if os.path.exists(device):
+            return device
+        # Durn, try adjusting the mapping
+        remapped = self._remap_device(os.path.basename(device))
+        if remapped:
+            LOG.debug("Remapped device name %s => %s", device, remapped)
+            return remapped
+        return None
+
+    def device_name_to_device(self, name):
+        # Translate a 'name' to a 'physical' device
+        if not name:
+            return None
+        # Try the ec2 mapping first
+        names = [name]
+        if name == 'root':
+            names.insert(0, 'ami')
+        if name == 'ami':
+            names.append('root')
+        device = None
+        LOG.debug("Using ec2 style lookup to find device %s", names)
+        for n in names:
+            device = self._ec2_name_to_device(n)
+            device = self._validate_device_name(device)
+            if device:
+                break
+        # Try the openstack way second
+        if not device:
+            LOG.debug("Using openstack style lookup to find device %s", names)
+            for n in names:
+                device = self._os_name_to_device(n)
+                device = self._validate_device_name(device)
+                if device:
+                    break
+        # Ok give up...
+        if not device:
+            return None
+        else:
+            LOG.debug("Mapped %s to device %s", name, device)
+            return device
+
+
+class BaseReader(object):
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, base_path):
+        self.base_path = base_path
+
+    @abc.abstractmethod
+    def _path_join(self, base, *add_ons):
+        pass
+
+    @abc.abstractmethod
+    def _path_exists(self, path):
+        pass
+
+    @abc.abstractmethod
+    def _path_read(self, path):
+        pass
+
+    @abc.abstractmethod
+    def _read_ec2_metadata(self):
+        pass
+
+    def _read_content_path(self, item):
+        path = item.get('content_path', '').lstrip("/")
+        path_pieces = path.split("/")
+        valid_pieces = [p for p in path_pieces if len(p)]
+        if not valid_pieces:
+            raise BrokenMetadata("Item %s has no valid content path" % (item))
+        path = self._path_join(self.base_path, "openstack", *path_pieces)
+        return self._path_read(path)
+
+    def _find_working_version(self, version):
+        search_versions = [version] + list(OS_VERSIONS)
+        for potential_version in search_versions:
+            if not potential_version:
+                continue
+            path = self._path_join(self.base_path, "openstack",
+                                   potential_version)
+            if self._path_exists(path):
+                if potential_version != version:
+                    LOG.warn("Version '%s' not available, attempting to use"
+                             " version '%s' instead", version,
+                             potential_version)
+                return potential_version
+        LOG.warn("Version '%s' not available, attempting to use '%s'"
+                 " instead", version, OS_LATEST)
+        return OS_LATEST
+
+    def read_v2(self, version=None):
+        """Reads a version 2 formatted location.
+
+        Return a dict with metadata, userdata, ec2-metadata, dsmode,
+        network_config, files and version (2).
+
+        If not a valid location, raise a NonReadable exception.
+        """
+
+        def datafiles(version):
+            files = {}
+            files['metadata'] = (
+                # File path to read
+                self._path_join("openstack", version, 'meta_data.json'),
+                # Is it required?
+                True,
+                # Translator function (applied after loading)
+                util.load_json,
+            )
+            files['userdata'] = (
+                self._path_join("openstack", version, 'user_data'),
+                False,
+                lambda x: x,
+            )
+            files['vendordata'] = (
+                self._path_join("openstack", version, 'vendor_data.json'),
+                False,
+                util.load_json,
+            )
+            return files
+
+        version = self._find_working_version(version)
+        results = {
+            'userdata': '',
+            'version': 2,
+        }
+        data = datafiles(version)
+        for (name, (path, required, translator)) in data.iteritems():
+            path = self._path_join(self.base_path, path)
+            data = None
+            found = False
+            if self._path_exists(path):
+                try:
+                    data = self._path_read(path)
+                except IOError:
+                    raise NonReadable("Failed to read: %s" % path)
+                found = True
+            else:
+                if required:
+                    raise NonReadable("Missing mandatory path: %s" % path)
+            if found and translator:
+                try:
+                    data = translator(data)
+                except Exception as e:
+                    raise BrokenMetadata("Failed to process "
+                                         "path %s: %s" % (path, e))
+            if found:
+                results[name] = data
+
+        metadata = results['metadata']
+        if 'random_seed' in metadata:
+            random_seed = metadata['random_seed']
+            try:
+                metadata['random_seed'] = base64.b64decode(random_seed)
+            except (ValueError, TypeError) as e:
+                raise BrokenMetadata("Badly formatted metadata"
+                                     " random_seed entry: %s" % e)
+
+        # load any files that were provided
+        files = {}
+        metadata_files = metadata.get('files', [])
+        for item in metadata_files:
+            if 'path' not in item:
+                continue
+            path = item['path']
+            try:
+                files[path] = self._read_content_path(item)
+            except Exception as e:
+                raise BrokenMetadata("Failed to read provided "
+                                     "file %s: %s" % (path, e))
+        results['files'] = files
+
+        # The 'network_config' item in metadata is a content pointer
+        # to the network config that should be applied. It is just a
+        # ubuntu/debian '/etc/network/interfaces' file.
+        net_item = metadata.get("network_config", None)
+        if net_item:
+            try:
+                results['network_config'] = self._read_content_path(net_item)
+            except IOError as e:
+                raise BrokenMetadata("Failed to read network"
+                                     " configuration: %s" % (e))
+
+        # To openstack, user can specify meta ('nova boot --meta=key=value')
+        # and those will appear under metadata['meta'].
+        # if they specify 'dsmode' they're indicating the mode that they intend
+        # for this datasource to operate in.
+        try:
+            results['dsmode'] = metadata['meta']['dsmode']
+        except KeyError:
+            pass
+
+        # Read any ec2-metadata (if applicable)
+        results['ec2-metadata'] = self._read_ec2_metadata()
+
+        # Perform some misc. metadata key renames...
+        for (target_key, source_key, is_required) in KEY_COPIES:
+            if is_required and source_key not in metadata:
+                raise BrokenMetadata("No '%s' entry in metadata" % source_key)
+            if source_key in metadata:
+                metadata[target_key] = metadata.get(source_key)
+        return results
+
+
+class ConfigDriveReader(BaseReader):
+    def __init__(self, base_path):
+        super(ConfigDriveReader, self).__init__(base_path)
+
+    def _path_join(self, base, *add_ons):
+        components = [base] + list(add_ons)
+        return os.path.join(*components)
+
+    def _path_exists(self, path):
+        return os.path.exists(path)
+
+    def _path_read(self, path):
+        return util.load_file(path)
+
+    def _read_ec2_metadata(self):
+        path = self._path_join(self.base_path,
+                               'ec2', 'latest', 'meta-data.json')
+        if not self._path_exists(path):
+            return {}
+        else:
+            try:
+                return util.load_json(self._path_read(path))
+            except Exception as e:
+                raise BrokenMetadata("Failed to process "
+                                     "path %s: %s" % (path, e))
+
+    def read_v1(self):
+        """Reads a version 1 formatted location.
+
+        Return a dict with metadata, userdata, dsmode, files and version (1).
+
+        If not a valid path, raise a NonReadable exception.
+        """
+
+        found = {}
+        for name in FILES_V1.keys():
+            path = self._path_join(self.base_path, name)
+            if self._path_exists(path):
+                found[name] = path
+        if len(found) == 0:
+            raise NonReadable("%s: no files found" % (self.base_path))
+
+        md = {}
+        for (name, (key, translator, default)) in FILES_V1.iteritems():
+            if name in found:
+                path = found[name]
+                try:
+                    contents = self._path_read(path)
+                except IOError:
+                    raise BrokenMetadata("Failed to read: %s" % path)
+                try:
+                    md[key] = translator(contents)
+                except Exception as e:
+                    raise BrokenMetadata("Failed to process "
+                                         "path %s: %s" % (path, e))
+            else:
+                md[key] = copy.deepcopy(default)
+
+        keydata = md['authorized_keys']
+        meta_js = md['meta_js']
+
+        # keydata in meta_js is preferred over "injected"
+        keydata = meta_js.get('public-keys', keydata)
+        if keydata:
+            lines = keydata.splitlines()
+            md['public-keys'] = [l for l in lines
+                                 if len(l) and not l.startswith("#")]
+
+        # config-drive-v1 has no way for openstack to provide the instance-id
+        # so we copy that into metadata from the user input
+        if 'instance-id' in meta_js:
+            md['instance-id'] = meta_js['instance-id']
+
+        results = {
+            'version': 1,
+            'metadata': md,
+        }
+
+        # allow the user to specify 'dsmode' in a meta tag
+        if 'dsmode' in meta_js:
+            results['dsmode'] = meta_js['dsmode']
+
+        # config-drive-v1 has no way of specifying user-data, so the user has
+        # to cheat and stuff it in a meta tag also.
+        results['userdata'] = meta_js.get('user-data', '')
+
+        # this implementation does not support files other than
+        # network/interfaces and authorized_keys...
+        results['files'] = {}
+
+        return results
+
+
+class MetadataReader(BaseReader):
+    def __init__(self, base_url, ssl_details=None, timeout=5, retries=5):
+        super(MetadataReader, self).__init__(base_url)
+        self._url_reader = functools.partial(url_helper.readurl,
+                                             retries=retries,
+                                             ssl_details=ssl_details,
+                                             timeout=timeout)
+        self._url_checker = functools.partial(url_helper.existsurl,
+                                              ssl_details=ssl_details,
+                                              timeout=timeout)
+        self._ec2_reader = functools.partial(ec2_utils.get_instance_metadata,
+                                             ssl_details=ssl_details,
+                                             timeout=timeout,
+                                             retries=retries)
+
+    def _path_read(self, path):
+        return str(self._url_reader(path))
+
+    def _path_exists(self, path):
+        return self._url_checker(path)
+
+    def _path_join(self, base, *add_ons):
+        return url_helper.combine_url(base, *add_ons)
+
+    def _read_ec2_metadata(self):
+        return self._ec2_reader()