diff options
| -rw-r--r-- | cloudinit/sources/DataSourceOpenStack.py | 163 | ||||
| -rw-r--r-- | cloudinit/sources/helpers/__init__.py | 14 | ||||
| -rw-r--r-- | cloudinit/sources/helpers/openstack.py | 420 | ||||
| -rw-r--r-- | tests/unittests/test_datasource/test_openstack.py | 122 |
4 files changed, 719 insertions, 0 deletions
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py new file mode 100644 index 00000000..44889f4e --- /dev/null +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -0,0 +1,163 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Yahoo! Inc. +# +# Author: Joshua Harlow <harlowja@yahoo-inc.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import time + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import url_helper +from cloudinit import util + +from cloudinit.sources.helpers import openstack + +LOG = logging.getLogger(__name__) + +# Various defaults/constants... +DEF_MD_URL = "http://169.254.169.254" +DEFAULT_IID = "iid-dsopenstack" +DEFAULT_METADATA = { + "instance-id": DEFAULT_IID, +} +VALID_DSMODES = ("net", "disabled") + + +class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths) + self.dsmode = 'net' + self.metadata_address = None + self.ssl_details = util.fetch_ssl_details(self.paths) + self.version = None + self.files = {} + + def __str__(self): + root = sources.DataSource.__str__(self) + mstr = "%s [%s,ver=%s]" % (root, self.dsmode, self.version) + return mstr + + def _get_url_settings(self): + # TODO(harlowja): this is shared with ec2 datasource, we should just + # move it to a shared location instead... + ds_cfg = self.ds_cfg + if not ds_cfg: + ds_cfg = {} + max_wait = 120 + try: + max_wait = int(ds_cfg.get("max_wait", max_wait)) + except Exception: + util.logexc(LOG, "Failed to get max wait. using %s", max_wait) + + timeout = 50 + try: + timeout = max(0, int(ds_cfg.get("timeout", timeout))) + except Exception: + util.logexc(LOG, "Failed to get timeout, using %s", timeout) + return (max_wait, timeout) + + def wait_for_metadata_service(self): + ds_cfg = self.ds_cfg + if not ds_cfg: + ds_cfg = {} + urls = ds_cfg.get("metadata_urls", [DEF_MD_URL]) + filtered = [x for x in urls if util.is_resolvable_url(x)] + if set(filtered) != set(urls): + LOG.debug("Removed the following from metadata urls: %s", + list((set(urls) - set(filtered)))) + if len(filtered): + urls = filtered + else: + LOG.warn("Empty metadata url list! using default list") + urls = [DEF_MD_URL] + + md_urls = [] + url2base = {} + for url in urls: + md_url = url_helper.combine_url(url, 'openstack', + openstack.OS_LATEST, + 'meta_data.json') + md_urls.append(md_url) + url2base[md_url] = url + + (max_wait, timeout) = self._get_url_settings() + if max_wait <= 0: + return False + start_time = time.time() + avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait, + timeout=timeout, + status_cb=LOG.warn) + if avail_url: + LOG.debug("Using metadata source: '%s'", url2base[avail_url]) + else: + LOG.critical("Giving up on md from %s after %s seconds", + md_urls, int(time.time() - start_time)) + + self.metadata_address = url2base.get(avail_url) + return bool(avail_url) + + def get_data(self): + try: + if not self.wait_for_metadata_service(): + return False + except IOError: + return False + + try: + results = util.log_time(LOG.debug, + 'Crawl of openstack metadata service', + read_metadata_service, + args=[self.metadata_address], + kwargs={'ssl_details': self.ssl_details}) + except openstack.NonReadable: + return False + except openstack.BrokenMetadata: + util.logexc(LOG, "Broken metadata address %s", + self.metadata_address) + return False + + user_dsmode = results.get('dsmode', None) + if user_dsmode not in VALID_DSMODES + (None,): + LOG.warn("User specified invalid mode: %s" % user_dsmode) + user_dsmode = None + if user_dsmode == 'disabled': + return False + + md = results.get('metadata', {}) + md = util.mergemanydict([md, DEFAULT_METADATA]) + self.metadata = md + self.ec2_metadata = results.get('ec2-metadata') + self.userdata_raw = results.get('userdata') + self.version = results['version'] + self.files.update(results.get('files', {})) + self.vendordata_raw = results.get('vendordata') + return True + + +def read_metadata_service(base_url, version=None, ssl_details=None): + reader = openstack.MetadataReader(base_url, ssl_details=ssl_details) + return reader.read_v2(version=version) + + +# Used to match classes to dependencies +datasources = [ + (DataSourceOpenStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/helpers/__init__.py b/cloudinit/sources/helpers/__init__.py new file mode 100644 index 00000000..2cf99ec8 --- /dev/null +++ b/cloudinit/sources/helpers/__init__.py @@ -0,0 +1,14 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py new file mode 100644 index 00000000..9dbef677 --- /dev/null +++ b/cloudinit/sources/helpers/openstack.py @@ -0,0 +1,420 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser <scott.moser@canonical.com> +# Author: Joshua Harlow <harlowja@yahoo-inc.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import abc +import base64 +import copy +import functools +import os + +from cloudinit import ec2_utils +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import url_helper +from cloudinit import util + +# For reference: http://tinyurl.com/laora4c + +LOG = logging.getLogger(__name__) + +FILES_V1 = { + # Path <-> (metadata key name, translator function, default value) + 'etc/network/interfaces': ('network_config', lambda x: x, ''), + 'meta.js': ('meta_js', util.load_json, {}), + "root/.ssh/authorized_keys": ('authorized_keys', lambda x: x, ''), +} +KEY_COPIES = ( + # Cloud-init metadata names <-> (metadata key, is required) + ('local-hostname', 'hostname', False), + ('instance-id', 'uuid', True), +) +OS_VERSIONS = ( + '2012-08-10', # folsom + '2013-04-04', # grizzly + '2013-10-17', # havana +) +OS_LATEST = 'latest' + + +class NonReadable(IOError): + pass + + +class BrokenMetadata(IOError): + pass + + +class SourceMixin(object): + def _ec2_name_to_device(self, name): + if not self.ec2_metadata: + return None + bdm = self.ec2_metadata.get('block-device-mapping', {}) + for (ent_name, device) in bdm.items(): + if name == ent_name: + return device + return None + + def get_public_ssh_keys(self): + name = "public_keys" + if self.version == 1: + name = "public-keys" + return sources.normalize_pubkey_data(self.metadata.get(name)) + + def _os_name_to_device(self, name): + device = None + try: + criteria = 'LABEL=%s' % (name) + if name == 'swap': + criteria = 'TYPE=%s' % (name) + dev_entries = util.find_devs_with(criteria) + if dev_entries: + device = dev_entries[0] + except util.ProcessExecutionError: + pass + return device + + def _validate_device_name(self, device): + if not device: + return None + if not device.startswith("/"): + device = "/dev/%s" % device + if os.path.exists(device): + return device + # Durn, try adjusting the mapping + remapped = self._remap_device(os.path.basename(device)) + if remapped: + LOG.debug("Remapped device name %s => %s", device, remapped) + return remapped + return None + + def device_name_to_device(self, name): + # Translate a 'name' to a 'physical' device + if not name: + return None + # Try the ec2 mapping first + names = [name] + if name == 'root': + names.insert(0, 'ami') + if name == 'ami': + names.append('root') + device = None + LOG.debug("Using ec2 style lookup to find device %s", names) + for n in names: + device = self._ec2_name_to_device(n) + device = self._validate_device_name(device) + if device: + break + # Try the openstack way second + if not device: + LOG.debug("Using openstack style lookup to find device %s", names) + for n in names: + device = self._os_name_to_device(n) + device = self._validate_device_name(device) + if device: + break + # Ok give up... + if not device: + return None + else: + LOG.debug("Mapped %s to device %s", name, device) + return device + + +class BaseReader(object): + __metaclass__ = abc.ABCMeta + + def __init__(self, base_path): + self.base_path = base_path + + @abc.abstractmethod + def _path_join(self, base, *add_ons): + pass + + @abc.abstractmethod + def _path_exists(self, path): + pass + + @abc.abstractmethod + def _path_read(self, path): + pass + + @abc.abstractmethod + def _read_ec2_metadata(self): + pass + + def _read_content_path(self, item): + path = item.get('content_path', '').lstrip("/") + path_pieces = path.split("/") + valid_pieces = [p for p in path_pieces if len(p)] + if not valid_pieces: + raise BrokenMetadata("Item %s has no valid content path" % (item)) + path = self._path_join(self.base_path, "openstack", *path_pieces) + return self._path_read(path) + + def _find_working_version(self, version): + search_versions = [version] + list(OS_VERSIONS) + for potential_version in search_versions: + if not potential_version: + continue + path = self._path_join(self.base_path, "openstack", + potential_version) + if self._path_exists(path): + if potential_version != version: + LOG.warn("Version '%s' not available, attempting to use" + " version '%s' instead", version, + potential_version) + return potential_version + LOG.warn("Version '%s' not available, attempting to use '%s'" + " instead", version, OS_LATEST) + return OS_LATEST + + def read_v2(self, version=None): + """Reads a version 2 formatted location. + + Return a dict with metadata, userdata, ec2-metadata, dsmode, + network_config, files and version (2). + + If not a valid location, raise a NonReadable exception. + """ + + def datafiles(version): + files = {} + files['metadata'] = ( + # File path to read + self._path_join("openstack", version, 'meta_data.json'), + # Is it required? + True, + # Translator function (applied after loading) + util.load_json, + ) + files['userdata'] = ( + self._path_join("openstack", version, 'user_data'), + False, + lambda x: x, + ) + files['vendordata'] = ( + self._path_join("openstack", version, 'vendor_data.json'), + False, + util.load_json, + ) + return files + + version = self._find_working_version(version) + results = { + 'userdata': '', + 'version': 2, + } + data = datafiles(version) + for (name, (path, required, translator)) in data.iteritems(): + path = self._path_join(self.base_path, path) + data = None + found = False + if self._path_exists(path): + try: + data = self._path_read(path) + except IOError: + raise NonReadable("Failed to read: %s" % path) + found = True + else: + if required: + raise NonReadable("Missing mandatory path: %s" % path) + if found and translator: + try: + data = translator(data) + except Exception as e: + raise BrokenMetadata("Failed to process " + "path %s: %s" % (path, e)) + if found: + results[name] = data + + metadata = results['metadata'] + if 'random_seed' in metadata: + random_seed = metadata['random_seed'] + try: + metadata['random_seed'] = base64.b64decode(random_seed) + except (ValueError, TypeError) as e: + raise BrokenMetadata("Badly formatted metadata" + " random_seed entry: %s" % e) + + # load any files that were provided + files = {} + metadata_files = metadata.get('files', []) + for item in metadata_files: + if 'path' not in item: + continue + path = item['path'] + try: + files[path] = self._read_content_path(item) + except Exception as e: + raise BrokenMetadata("Failed to read provided " + "file %s: %s" % (path, e)) + results['files'] = files + + # The 'network_config' item in metadata is a content pointer + # to the network config that should be applied. It is just a + # ubuntu/debian '/etc/network/interfaces' file. + net_item = metadata.get("network_config", None) + if net_item: + try: + results['network_config'] = self._read_content_path(net_item) + except IOError as e: + raise BrokenMetadata("Failed to read network" + " configuration: %s" % (e)) + + # To openstack, user can specify meta ('nova boot --meta=key=value') + # and those will appear under metadata['meta']. + # if they specify 'dsmode' they're indicating the mode that they intend + # for this datasource to operate in. + try: + results['dsmode'] = metadata['meta']['dsmode'] + except KeyError: + pass + + # Read any ec2-metadata (if applicable) + results['ec2-metadata'] = self._read_ec2_metadata() + + # Perform some misc. metadata key renames... + for (target_key, source_key, is_required) in KEY_COPIES: + if is_required and source_key not in metadata: + raise BrokenMetadata("No '%s' entry in metadata" % source_key) + if source_key in metadata: + metadata[target_key] = metadata.get(source_key) + return results + + +class ConfigDriveReader(BaseReader): + def __init__(self, base_path): + super(ConfigDriveReader, self).__init__(base_path) + + def _path_join(self, base, *add_ons): + components = [base] + list(add_ons) + return os.path.join(*components) + + def _path_exists(self, path): + return os.path.exists(path) + + def _path_read(self, path): + return util.load_file(path) + + def _read_ec2_metadata(self): + path = self._path_join(self.base_path, + 'ec2', 'latest', 'meta-data.json') + if not self._path_exists(path): + return {} + else: + try: + return util.load_json(self._path_read(path)) + except Exception as e: + raise BrokenMetadata("Failed to process " + "path %s: %s" % (path, e)) + + def read_v1(self): + """Reads a version 1 formatted location. + + Return a dict with metadata, userdata, dsmode, files and version (1). + + If not a valid path, raise a NonReadable exception. + """ + + found = {} + for name in FILES_V1.keys(): + path = self._path_join(self.base_path, name) + if self._path_exists(path): + found[name] = path + if len(found) == 0: + raise NonReadable("%s: no files found" % (self.base_path)) + + md = {} + for (name, (key, translator, default)) in FILES_V1.iteritems(): + if name in found: + path = found[name] + try: + contents = self._path_read(path) + except IOError: + raise BrokenMetadata("Failed to read: %s" % path) + try: + md[key] = translator(contents) + except Exception as e: + raise BrokenMetadata("Failed to process " + "path %s: %s" % (path, e)) + else: + md[key] = copy.deepcopy(default) + + keydata = md['authorized_keys'] + meta_js = md['meta_js'] + + # keydata in meta_js is preferred over "injected" + keydata = meta_js.get('public-keys', keydata) + if keydata: + lines = keydata.splitlines() + md['public-keys'] = [l for l in lines + if len(l) and not l.startswith("#")] + + # config-drive-v1 has no way for openstack to provide the instance-id + # so we copy that into metadata from the user input + if 'instance-id' in meta_js: + md['instance-id'] = meta_js['instance-id'] + + results = { + 'version': 1, + 'metadata': md, + } + + # allow the user to specify 'dsmode' in a meta tag + if 'dsmode' in meta_js: + results['dsmode'] = meta_js['dsmode'] + + # config-drive-v1 has no way of specifying user-data, so the user has + # to cheat and stuff it in a meta tag also. + results['userdata'] = meta_js.get('user-data', '') + + # this implementation does not support files other than + # network/interfaces and authorized_keys... + results['files'] = {} + + return results + + +class MetadataReader(BaseReader): + def __init__(self, base_url, ssl_details=None, timeout=5, retries=5): + super(MetadataReader, self).__init__(base_url) + self._url_reader = functools.partial(url_helper.readurl, + retries=retries, + ssl_details=ssl_details, + timeout=timeout) + self._url_checker = functools.partial(url_helper.existsurl, + ssl_details=ssl_details, + timeout=timeout) + self._ec2_reader = functools.partial(ec2_utils.get_instance_metadata, + ssl_details=ssl_details, + timeout=timeout, + retries=retries) + + def _path_read(self, path): + return str(self._url_reader(path)) + + def _path_exists(self, path): + return self._url_checker(path) + + def _path_join(self, base, *add_ons): + return url_helper.combine_url(base, *add_ons) + + def _read_ec2_metadata(self): + return self._ec2_reader() diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py new file mode 100644 index 00000000..7d93f1d3 --- /dev/null +++ b/tests/unittests/test_datasource/test_openstack.py @@ -0,0 +1,122 @@ +import re +import json + +from StringIO import StringIO + +from urlparse import urlparse + +from tests.unittests import helpers + +from cloudinit.sources import DataSourceOpenStack as ds +from cloudinit.sources.helpers import openstack +from cloudinit import util + +import httpretty as hp + +BASE_URL = "http://169.254.169.254" +PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' +EC2_META = { + 'ami-id': 'ami-00000001', + 'ami-launch-index': 0, + 'ami-manifest-path': 'FIXME', + 'hostname': 'sm-foo-test.novalocal', + 'instance-action': 'none', + 'instance-id': 'i-00000001', + 'instance-type': 'm1.tiny', + 'local-hostname': 'sm-foo-test.novalocal', + 'local-ipv4': '0.0.0.0', + 'public-hostname': 'sm-foo-test.novalocal', + 'public-ipv4': '0.0.0.1', + 'reservation-id': 'r-iru5qm4m', +} +USER_DATA = '#!/bin/sh\necho This is user data\n' +VENDOR_DATA = { + 'magic': '', +} +OSTACK_META = { + 'availability_zone': 'nova', + 'files': [{'content_path': '/content/0000', 'path': '/etc/foo.cfg'}, + {'content_path': '/content/0001', 'path': '/etc/bar/bar.cfg'}], + 'hostname': 'sm-foo-test.novalocal', + 'meta': {'dsmode': 'local', 'my-meta': 'my-value'}, + 'name': 'sm-foo-test', + 'public_keys': {'mykey': PUBKEY}, + 'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c', +} +CONTENT_0 = 'This is contents of /etc/foo.cfg\n' +CONTENT_1 = '# this is /etc/bar/bar.cfg\n' +OS_FILES = { + 'openstack/2012-08-10/meta_data.json': json.dumps(OSTACK_META), + 'openstack/2012-08-10/user_data': USER_DATA, + 'openstack/content/0000': CONTENT_0, + 'openstack/content/0001': CONTENT_1, + 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), + 'openstack/latest/user_data': USER_DATA, + 'openstack/latest/vendor_data.json': json.dumps(VENDOR_DATA), +} +EC2_FILES = { + 'latest/user-data': USER_DATA, +} + + +def _register_uris(version): + + def match_ec2_url(uri, headers): + path = uri.path.lstrip("/") + if path in EC2_FILES: + return (200, headers, EC2_FILES.get(path)) + if path == 'latest/meta-data': + buf = StringIO() + for (k, v) in EC2_META.items(): + if isinstance(v, (list, tuple)): + buf.write("%s/" % (k)) + else: + buf.write("%s" % (k)) + buf.write("\n") + return (200, headers, buf.getvalue()) + if path.startswith('latest/meta-data'): + value = None + pieces = path.split("/") + if path.endswith("/"): + pieces = pieces[2:-1] + value = util.get_cfg_by_path(EC2_META, pieces) + else: + pieces = pieces[2:] + value = util.get_cfg_by_path(EC2_META, pieces) + if value is not None: + return (200, headers, str(value)) + return (404, headers, '') + + def get_request_callback(method, uri, headers): + uri = urlparse(uri) + path = uri.path.lstrip("/") + if path in OS_FILES: + return (200, headers, OS_FILES.get(path)) + return match_ec2_url(uri, headers) + + def head_request_callback(method, uri, headers): + uri = urlparse(uri) + path = uri.path.lstrip("/") + for key in OS_FILES.keys(): + if key.startswith(path): + return (200, headers, '') + return (404, headers, '') + + hp.register_uri(hp.GET, re.compile(r'http://169.254.169.254/.*'), + body=get_request_callback) + + hp.register_uri(hp.HEAD, re.compile(r'http://169.254.169.254/.*'), + body=head_request_callback) + + +class TestOpenStackDataSource(helpers.TestCase): + VERSION = 'latest' + + @hp.activate + def test_fetch(self): + _register_uris(self.VERSION) + f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + self.assertEquals(VENDOR_DATA, f.get('vendordata')) + self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertEquals(USER_DATA, f.get('userdata')) |