From c104d6dfa464a8906c16b4f09b4b76ab5bf2e4e1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 1 Feb 2014 22:48:55 -0800 Subject: Add a openstack specific datasource Openstack has a unique derivative datasource that is gaining usage. Previously the config drive datasource provided part of this functionality as well as the ec2 datasource, but since new functionality is being added to openstack is seems benefical to combine the used parts into one datasource just made for handling openstack deployments. This patch factors out the common logic shared between the config drive and the openstack metadata datasource and places that in a shared helper file and then creates a new openstack datasource that readers from the openstack metadata service and refactors the config drive datasource to use this common logic. --- cloudinit/sources/DataSourceConfigDrive.py | 463 +++++++---------------------- 1 file changed, 108 insertions(+), 355 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 4f437244..c45a1119 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -18,181 +18,79 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import base64 -import json import os from cloudinit import log as logging from cloudinit import sources from cloudinit import util +from cloudinit.sources.helpers import openstack + LOG = logging.getLogger(__name__) # Various defaults/constants... DEFAULT_IID = "iid-dsconfigdrive" DEFAULT_MODE = 'pass' -CFG_DRIVE_FILES_V1 = [ - "etc/network/interfaces", - "root/.ssh/authorized_keys", - "meta.js", -] DEFAULT_METADATA = { "instance-id": DEFAULT_IID, } VALID_DSMODES = ("local", "net", "pass", "disabled") +FS_TYPES = ('vfat', 'iso9660') +LABEL_TYPES = ('config-2',) +OPTICAL_DEVICES = tuple(('/dev/sr%s' % i for i in range(0, 2))) -class ConfigDriveHelper(object): - def __init__(self, distro): - self.distro = distro - - def on_first_boot(self, data): - if not data: - data = {} - if 'network_config' in data: - LOG.debug("Updating network interfaces from config drive") - self.distro.apply_network(data['network_config']) - files = data.get('files') - if files: - LOG.debug("Writing %s injected files", len(files)) - try: - write_files(files) - except IOError: - util.logexc(LOG, "Failed writing files") - - -class DataSourceConfigDrive(sources.DataSource): +class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) + super(DataSourceConfigDrive, self).__init__(sys_cfg, distro, paths) self.source = None self.dsmode = 'local' self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') self.version = None self.ec2_metadata = None - self.helper = ConfigDriveHelper(distro) + self.files = {} def __str__(self): root = sources.DataSource.__str__(self) - mstr = "%s [%s,ver=%s]" % (root, - self.dsmode, - self.version) + mstr = "%s [%s,ver=%s]" % (root, self.dsmode, self.version) mstr += "[source=%s]" % (self.source) return mstr - def _ec2_name_to_device(self, name): - if not self.ec2_metadata: - return None - bdm = self.ec2_metadata.get('block-device-mapping', {}) - for (ent_name, device) in bdm.items(): - if name == ent_name: - return device - return None - - def _os_name_to_device(self, name): - device = None - try: - criteria = 'LABEL=%s' % (name) - if name in ['swap']: - criteria = 'TYPE=%s' % (name) - dev_entries = util.find_devs_with(criteria) - if dev_entries: - device = dev_entries[0] - except util.ProcessExecutionError: - pass - return device - - def _validate_device_name(self, device): - if not device: - return None - if not device.startswith("/"): - device = "/dev/%s" % device - if os.path.exists(device): - return device - # Durn, try adjusting the mapping - remapped = self._remap_device(os.path.basename(device)) - if remapped: - LOG.debug("Remapped device name %s => %s", device, remapped) - return remapped - return None - - def device_name_to_device(self, name): - # Translate a 'name' to a 'physical' device - if not name: - return None - # Try the ec2 mapping first - names = [name] - if name == 'root': - names.insert(0, 'ami') - if name == 'ami': - names.append('root') - device = None - LOG.debug("Using ec2 metadata lookup to find device %s", names) - for n in names: - device = self._ec2_name_to_device(n) - device = self._validate_device_name(device) - if device: - break - # Try the openstack way second - if not device: - LOG.debug("Using os lookup to find device %s", names) - for n in names: - device = self._os_name_to_device(n) - device = self._validate_device_name(device) - if device: - break - # Ok give up... - if not device: - return None - else: - LOG.debug("Using cfg drive lookup mapped to device %s", device) - return device - def get_data(self): found = None md = {} - results = {} if os.path.isdir(self.seed_dir): try: - results = read_config_drive_dir(self.seed_dir) + results = read_config_drive(self.seed_dir) found = self.seed_dir - except NonConfigDriveDir: + except openstack.NonReadable: util.logexc(LOG, "Failed reading config drive from %s", self.seed_dir) if not found: - devlist = find_candidate_devs() - for dev in devlist: + for dev in find_candidate_devs(): try: - results = util.mount_cb(dev, read_config_drive_dir) + results = util.mount_cb(dev, read_config_drive) found = dev - break - except (NonConfigDriveDir, util.MountFailedError): + except openstack.NonReadable: pass - except BrokenConfigDriveDir: - util.logexc(LOG, "broken config drive: %s", dev) - + except util.MountFailedError: + pass + except openstack.BrokenMetadata: + util.logexc(LOG, "Broken config drive: %s", dev) + if found: + break if not found: return False - md = results['metadata'] + md = results.get('metadata', {}) md = util.mergemanydict([md, DEFAULT_METADATA]) - - # Perform some metadata 'fixups' - # - # OpenStack uses the 'hostname' key - # while most of cloud-init uses the metadata - # 'local-hostname' key instead so if it doesn't - # exist we need to make sure its copied over. - for (tgt, src) in [('local-hostname', 'hostname')]: - if tgt not in md and src in md: - md[tgt] = md[src] - user_dsmode = results.get('dsmode', None) if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("user specified invalid mode: %s" % user_dsmode) + LOG.warn("User specified invalid mode: %s" % user_dsmode) user_dsmode = None - dsmode = get_ds_mode(cfgdrv_ver=results['cfgdrive_ver'], + dsmode = get_ds_mode(cfgdrv_ver=results['version'], ds_cfg=self.ds_cfg.get('dsmode'), user=user_dsmode) @@ -209,7 +107,7 @@ class DataSourceConfigDrive(sources.DataSource): prev_iid = get_previous_iid(self.paths) cur_iid = md['instance-id'] if prev_iid != cur_iid and self.dsmode == "local": - self.helper.on_first_boot(results) + on_first_boot(results, distro=self.distro) # dsmode != self.dsmode here if: # * dsmode = "pass", pass means it should only copy files and then @@ -225,16 +123,11 @@ class DataSourceConfigDrive(sources.DataSource): self.metadata = md self.ec2_metadata = results.get('ec2-metadata') self.userdata_raw = results.get('userdata') - self.version = results['cfgdrive_ver'] - + self.version = results['version'] + self.files.update(results.get('files', {})) + self.vendordata_raw = results.get('vendordata') return True - def get_public_ssh_keys(self): - name = "public_keys" - if self.version == 1: - name = "public-keys" - return sources.normalize_pubkey_data(self.metadata.get(name)) - class DataSourceConfigDriveNet(DataSourceConfigDrive): def __init__(self, sys_cfg, distro, paths): @@ -242,220 +135,6 @@ class DataSourceConfigDriveNet(DataSourceConfigDrive): self.dsmode = 'net' -class NonConfigDriveDir(Exception): - pass - - -class BrokenConfigDriveDir(Exception): - pass - - -def find_candidate_devs(): - """Return a list of devices that may contain the config drive. - - The returned list is sorted by search order where the first item has - should be searched first (highest priority) - - config drive v1: - Per documentation, this is "associated as the last available disk on the - instance", and should be VFAT. - Currently, we do not restrict search list to "last available disk" - - config drive v2: - Disk should be: - * either vfat or iso9660 formated - * labeled with 'config-2' - """ - - # Query optical drive to get it in blkid cache for 2.6 kernels - util.find_devs_with(path="/dev/sr0") - util.find_devs_with(path="/dev/sr1") - - by_fstype = (util.find_devs_with("TYPE=vfat") + - util.find_devs_with("TYPE=iso9660")) - by_label = util.find_devs_with("LABEL=config-2") - - # give preference to "last available disk" (vdb over vda) - # note, this is not a perfect rendition of that. - by_fstype.sort(reverse=True) - by_label.sort(reverse=True) - - # combine list of items by putting by-label items first - # followed by fstype items, but with dupes removed - combined = (by_label + [d for d in by_fstype if d not in by_label]) - - # We are looking for block device (sda, not sda1), ignore partitions - combined = [d for d in combined if not util.is_partition(d)] - - return combined - - -def read_config_drive_dir(source_dir): - last_e = NonConfigDriveDir("Not found") - for finder in (read_config_drive_dir_v2, read_config_drive_dir_v1): - try: - data = finder(source_dir) - return data - except NonConfigDriveDir as exc: - last_e = exc - raise last_e - - -def read_config_drive_dir_v2(source_dir, version="2012-08-10"): - - if (not os.path.isdir(os.path.join(source_dir, "openstack", version)) and - os.path.isdir(os.path.join(source_dir, "openstack", "latest"))): - LOG.warn("version '%s' not available, attempting to use 'latest'" % - version) - version = "latest" - - datafiles = ( - ('metadata', - "openstack/%s/meta_data.json" % version, True, json.loads), - ('userdata', "openstack/%s/user_data" % version, False, None), - ('ec2-metadata', "ec2/latest/meta-data.json", False, json.loads), - ) - - results = {'userdata': None} - for (name, path, required, process) in datafiles: - fpath = os.path.join(source_dir, path) - data = None - found = False - if os.path.isfile(fpath): - try: - data = util.load_file(fpath) - except IOError: - raise BrokenConfigDriveDir("Failed to read: %s" % fpath) - found = True - elif required: - raise NonConfigDriveDir("Missing mandatory path: %s" % fpath) - - if found and process: - try: - data = process(data) - except Exception as exc: - raise BrokenConfigDriveDir(("Failed to process " - "path: %s") % fpath) - - if found: - results[name] = data - - # instance-id is 'uuid' for openstack. just copy it to instance-id. - if 'instance-id' not in results['metadata']: - try: - results['metadata']['instance-id'] = results['metadata']['uuid'] - except KeyError: - raise BrokenConfigDriveDir("No uuid entry in metadata") - - if 'random_seed' in results['metadata']: - random_seed = results['metadata']['random_seed'] - try: - results['metadata']['random_seed'] = base64.b64decode(random_seed) - except (ValueError, TypeError) as exc: - raise BrokenConfigDriveDir("Badly formatted random_seed: %s" % exc) - - def read_content_path(item): - # do not use os.path.join here, as content_path starts with / - cpath = os.path.sep.join((source_dir, "openstack", - "./%s" % item['content_path'])) - return util.load_file(cpath) - - files = {} - try: - for item in results['metadata'].get('files', {}): - files[item['path']] = read_content_path(item) - - # the 'network_config' item in metadata is a content pointer - # to the network config that should be applied. - # in folsom, it is just a '/etc/network/interfaces' file. - item = results['metadata'].get("network_config", None) - if item: - results['network_config'] = read_content_path(item) - except Exception as exc: - raise BrokenConfigDriveDir("Failed to read file %s: %s" % (item, exc)) - - # to openstack, user can specify meta ('nova boot --meta=key=value') and - # those will appear under metadata['meta']. - # if they specify 'dsmode' they're indicating the mode that they intend - # for this datasource to operate in. - try: - results['dsmode'] = results['metadata']['meta']['dsmode'] - except KeyError: - pass - - results['files'] = files - results['cfgdrive_ver'] = 2 - return results - - -def read_config_drive_dir_v1(source_dir): - """ - read source_dir, and return a tuple with metadata dict, user-data, - files and version (1). If not a valid dir, raise a NonConfigDriveDir - """ - - found = {} - for af in CFG_DRIVE_FILES_V1: - fn = os.path.join(source_dir, af) - if os.path.isfile(fn): - found[af] = fn - - if len(found) == 0: - raise NonConfigDriveDir("%s: %s" % (source_dir, "no files found")) - - md = {} - keydata = "" - if "etc/network/interfaces" in found: - fn = found["etc/network/interfaces"] - md['network_config'] = util.load_file(fn) - - if "root/.ssh/authorized_keys" in found: - fn = found["root/.ssh/authorized_keys"] - keydata = util.load_file(fn) - - meta_js = {} - if "meta.js" in found: - fn = found['meta.js'] - content = util.load_file(fn) - try: - # Just check if its really json... - meta_js = json.loads(content) - if not isinstance(meta_js, (dict)): - raise TypeError("Dict expected for meta.js root node") - except (ValueError, TypeError) as e: - raise NonConfigDriveDir("%s: %s, %s" % - (source_dir, "invalid json in meta.js", e)) - md['meta_js'] = content - - # keydata in meta_js is preferred over "injected" - keydata = meta_js.get('public-keys', keydata) - if keydata: - lines = keydata.splitlines() - md['public-keys'] = [l for l in lines - if len(l) and not l.startswith("#")] - - # config-drive-v1 has no way for openstack to provide the instance-id - # so we copy that into metadata from the user input - if 'instance-id' in meta_js: - md['instance-id'] = meta_js['instance-id'] - - results = {'cfgdrive_ver': 1, 'metadata': md} - - # allow the user to specify 'dsmode' in a meta tag - if 'dsmode' in meta_js: - results['dsmode'] = meta_js['dsmode'] - - # config-drive-v1 has no way of specifying user-data, so the user has - # to cheat and stuff it in a meta tag also. - results['userdata'] = meta_js.get('user-data') - - # this implementation does not support files - # (other than network/interfaces and authorized_keys) - results['files'] = [] - - return results - - def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): """Determine what mode should be used. valid values are 'pass', 'disabled', 'local', 'net' @@ -481,6 +160,21 @@ def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): return "net" +def read_config_drive(source_dir, version="2012-08-10"): + reader = openstack.ConfigDriveReader(source_dir) + finders = [ + (reader.read_v2, [], {'version': version}), + (reader.read_v1, [], {}), + ] + excps = [] + for (functor, args, kwargs) in finders: + try: + return functor(*args, **kwargs) + except openstack.NonReadable as e: + excps.append(e) + raise excps[-1] + + def get_previous_iid(paths): # interestingly, for this purpose the "previous" instance-id is the current # instance-id. cloud-init hasn't moved them over yet as this datasource @@ -492,17 +186,76 @@ def get_previous_iid(paths): return None -def write_files(files): - for (name, content) in files.iteritems(): - if name[0] != os.sep: - name = os.sep + name - util.write_file(name, content, mode=0660) +def on_first_boot(data, distro=None): + """Performs any first-boot actions using data read from a config-drive.""" + if not isinstance(data, dict): + raise TypeError("Config-drive data expected to be a dict; not %s" + % (type(data))) + net_conf = data.get("network_config", '') + if net_conf and distro: + LOG.debug("Updating network interfaces from config drive") + distro.apply_network(net_conf) + files = data.get('files', {}) + if files: + LOG.debug("Writing %s injected files", len(files)) + for (filename, content) in files.iteritems(): + if not filename.startswith(os.sep): + filename = os.sep + filename + try: + util.write_file(filename, content, mode=0660) + except IOError: + util.logexc(LOG, "Failed writing file: %s", filename) + + +def find_candidate_devs(probe_optical=True): + """Return a list of devices that may contain the config drive. + + The returned list is sorted by search order where the first item has + should be searched first (highest priority) + + config drive v1: + Per documentation, this is "associated as the last available disk on the + instance", and should be VFAT. + Currently, we do not restrict search list to "last available disk" + + config drive v2: + Disk should be: + * either vfat or iso9660 formated + * labeled with 'config-2' + """ + # query optical drive to get it in blkid cache for 2.6 kernels + if probe_optical: + for device in OPTICAL_DEVICES: + try: + util.find_devs_with(path=device) + except util.ProcessExecutionError: + pass + + by_fstype = [] + for fs_type in FS_TYPES: + by_fstype.extend(util.find_devs_with("TYPE=%s" % (fs_type))) + + by_label = [] + for label in LABEL_TYPES: + by_label.extend(util.find_devs_with("LABEL=%s" % (label))) + + # give preference to "last available disk" (vdb over vda) + # note, this is not a perfect rendition of that. + by_fstype.sort(reverse=True) + by_label.sort(reverse=True) + + # combine list of items by putting by-label items first + # followed by fstype items, but with dupes removed + combined = (by_label + [d for d in by_fstype if d not in by_label]) + + # we are looking for block device (sda, not sda1), ignore partitions + return [d for d in combined if not util.is_partition(d)] # Used to match classes to dependencies datasources = [ - (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), - (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), + (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] -- cgit v1.2.3 From 810df2c55c108e7e4064263e508d9786d8b1dc8e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 1 Feb 2014 22:58:58 -0800 Subject: Don't forget the rest of the files! --- cloudinit/sources/DataSourceOpenStack.py | 163 +++++++++ cloudinit/sources/helpers/__init__.py | 14 + cloudinit/sources/helpers/openstack.py | 420 ++++++++++++++++++++++ tests/unittests/test_datasource/test_openstack.py | 122 +++++++ 4 files changed, 719 insertions(+) create mode 100644 cloudinit/sources/DataSourceOpenStack.py create mode 100644 cloudinit/sources/helpers/__init__.py create mode 100644 cloudinit/sources/helpers/openstack.py create mode 100644 tests/unittests/test_datasource/test_openstack.py (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py new file mode 100644 index 00000000..44889f4e --- /dev/null +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -0,0 +1,163 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import time + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import url_helper +from cloudinit import util + +from cloudinit.sources.helpers import openstack + +LOG = logging.getLogger(__name__) + +# Various defaults/constants... +DEF_MD_URL = "http://169.254.169.254" +DEFAULT_IID = "iid-dsopenstack" +DEFAULT_METADATA = { + "instance-id": DEFAULT_IID, +} +VALID_DSMODES = ("net", "disabled") + + +class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths) + self.dsmode = 'net' + self.metadata_address = None + self.ssl_details = util.fetch_ssl_details(self.paths) + self.version = None + self.files = {} + + def __str__(self): + root = sources.DataSource.__str__(self) + mstr = "%s [%s,ver=%s]" % (root, self.dsmode, self.version) + return mstr + + def _get_url_settings(self): + # TODO(harlowja): this is shared with ec2 datasource, we should just + # move it to a shared location instead... + ds_cfg = self.ds_cfg + if not ds_cfg: + ds_cfg = {} + max_wait = 120 + try: + max_wait = int(ds_cfg.get("max_wait", max_wait)) + except Exception: + util.logexc(LOG, "Failed to get max wait. using %s", max_wait) + + timeout = 50 + try: + timeout = max(0, int(ds_cfg.get("timeout", timeout))) + except Exception: + util.logexc(LOG, "Failed to get timeout, using %s", timeout) + return (max_wait, timeout) + + def wait_for_metadata_service(self): + ds_cfg = self.ds_cfg + if not ds_cfg: + ds_cfg = {} + urls = ds_cfg.get("metadata_urls", [DEF_MD_URL]) + filtered = [x for x in urls if util.is_resolvable_url(x)] + if set(filtered) != set(urls): + LOG.debug("Removed the following from metadata urls: %s", + list((set(urls) - set(filtered)))) + if len(filtered): + urls = filtered + else: + LOG.warn("Empty metadata url list! using default list") + urls = [DEF_MD_URL] + + md_urls = [] + url2base = {} + for url in urls: + md_url = url_helper.combine_url(url, 'openstack', + openstack.OS_LATEST, + 'meta_data.json') + md_urls.append(md_url) + url2base[md_url] = url + + (max_wait, timeout) = self._get_url_settings() + if max_wait <= 0: + return False + start_time = time.time() + avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait, + timeout=timeout, + status_cb=LOG.warn) + if avail_url: + LOG.debug("Using metadata source: '%s'", url2base[avail_url]) + else: + LOG.critical("Giving up on md from %s after %s seconds", + md_urls, int(time.time() - start_time)) + + self.metadata_address = url2base.get(avail_url) + return bool(avail_url) + + def get_data(self): + try: + if not self.wait_for_metadata_service(): + return False + except IOError: + return False + + try: + results = util.log_time(LOG.debug, + 'Crawl of openstack metadata service', + read_metadata_service, + args=[self.metadata_address], + kwargs={'ssl_details': self.ssl_details}) + except openstack.NonReadable: + return False + except openstack.BrokenMetadata: + util.logexc(LOG, "Broken metadata address %s", + self.metadata_address) + return False + + user_dsmode = results.get('dsmode', None) + if user_dsmode not in VALID_DSMODES + (None,): + LOG.warn("User specified invalid mode: %s" % user_dsmode) + user_dsmode = None + if user_dsmode == 'disabled': + return False + + md = results.get('metadata', {}) + md = util.mergemanydict([md, DEFAULT_METADATA]) + self.metadata = md + self.ec2_metadata = results.get('ec2-metadata') + self.userdata_raw = results.get('userdata') + self.version = results['version'] + self.files.update(results.get('files', {})) + self.vendordata_raw = results.get('vendordata') + return True + + +def read_metadata_service(base_url, version=None, ssl_details=None): + reader = openstack.MetadataReader(base_url, ssl_details=ssl_details) + return reader.read_v2(version=version) + + +# Used to match classes to dependencies +datasources = [ + (DataSourceOpenStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/helpers/__init__.py b/cloudinit/sources/helpers/__init__.py new file mode 100644 index 00000000..2cf99ec8 --- /dev/null +++ b/cloudinit/sources/helpers/__init__.py @@ -0,0 +1,14 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py new file mode 100644 index 00000000..9dbef677 --- /dev/null +++ b/cloudinit/sources/helpers/openstack.py @@ -0,0 +1,420 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import abc +import base64 +import copy +import functools +import os + +from cloudinit import ec2_utils +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import url_helper +from cloudinit import util + +# For reference: http://tinyurl.com/laora4c + +LOG = logging.getLogger(__name__) + +FILES_V1 = { + # Path <-> (metadata key name, translator function, default value) + 'etc/network/interfaces': ('network_config', lambda x: x, ''), + 'meta.js': ('meta_js', util.load_json, {}), + "root/.ssh/authorized_keys": ('authorized_keys', lambda x: x, ''), +} +KEY_COPIES = ( + # Cloud-init metadata names <-> (metadata key, is required) + ('local-hostname', 'hostname', False), + ('instance-id', 'uuid', True), +) +OS_VERSIONS = ( + '2012-08-10', # folsom + '2013-04-04', # grizzly + '2013-10-17', # havana +) +OS_LATEST = 'latest' + + +class NonReadable(IOError): + pass + + +class BrokenMetadata(IOError): + pass + + +class SourceMixin(object): + def _ec2_name_to_device(self, name): + if not self.ec2_metadata: + return None + bdm = self.ec2_metadata.get('block-device-mapping', {}) + for (ent_name, device) in bdm.items(): + if name == ent_name: + return device + return None + + def get_public_ssh_keys(self): + name = "public_keys" + if self.version == 1: + name = "public-keys" + return sources.normalize_pubkey_data(self.metadata.get(name)) + + def _os_name_to_device(self, name): + device = None + try: + criteria = 'LABEL=%s' % (name) + if name == 'swap': + criteria = 'TYPE=%s' % (name) + dev_entries = util.find_devs_with(criteria) + if dev_entries: + device = dev_entries[0] + except util.ProcessExecutionError: + pass + return device + + def _validate_device_name(self, device): + if not device: + return None + if not device.startswith("/"): + device = "/dev/%s" % device + if os.path.exists(device): + return device + # Durn, try adjusting the mapping + remapped = self._remap_device(os.path.basename(device)) + if remapped: + LOG.debug("Remapped device name %s => %s", device, remapped) + return remapped + return None + + def device_name_to_device(self, name): + # Translate a 'name' to a 'physical' device + if not name: + return None + # Try the ec2 mapping first + names = [name] + if name == 'root': + names.insert(0, 'ami') + if name == 'ami': + names.append('root') + device = None + LOG.debug("Using ec2 style lookup to find device %s", names) + for n in names: + device = self._ec2_name_to_device(n) + device = self._validate_device_name(device) + if device: + break + # Try the openstack way second + if not device: + LOG.debug("Using openstack style lookup to find device %s", names) + for n in names: + device = self._os_name_to_device(n) + device = self._validate_device_name(device) + if device: + break + # Ok give up... + if not device: + return None + else: + LOG.debug("Mapped %s to device %s", name, device) + return device + + +class BaseReader(object): + __metaclass__ = abc.ABCMeta + + def __init__(self, base_path): + self.base_path = base_path + + @abc.abstractmethod + def _path_join(self, base, *add_ons): + pass + + @abc.abstractmethod + def _path_exists(self, path): + pass + + @abc.abstractmethod + def _path_read(self, path): + pass + + @abc.abstractmethod + def _read_ec2_metadata(self): + pass + + def _read_content_path(self, item): + path = item.get('content_path', '').lstrip("/") + path_pieces = path.split("/") + valid_pieces = [p for p in path_pieces if len(p)] + if not valid_pieces: + raise BrokenMetadata("Item %s has no valid content path" % (item)) + path = self._path_join(self.base_path, "openstack", *path_pieces) + return self._path_read(path) + + def _find_working_version(self, version): + search_versions = [version] + list(OS_VERSIONS) + for potential_version in search_versions: + if not potential_version: + continue + path = self._path_join(self.base_path, "openstack", + potential_version) + if self._path_exists(path): + if potential_version != version: + LOG.warn("Version '%s' not available, attempting to use" + " version '%s' instead", version, + potential_version) + return potential_version + LOG.warn("Version '%s' not available, attempting to use '%s'" + " instead", version, OS_LATEST) + return OS_LATEST + + def read_v2(self, version=None): + """Reads a version 2 formatted location. + + Return a dict with metadata, userdata, ec2-metadata, dsmode, + network_config, files and version (2). + + If not a valid location, raise a NonReadable exception. + """ + + def datafiles(version): + files = {} + files['metadata'] = ( + # File path to read + self._path_join("openstack", version, 'meta_data.json'), + # Is it required? + True, + # Translator function (applied after loading) + util.load_json, + ) + files['userdata'] = ( + self._path_join("openstack", version, 'user_data'), + False, + lambda x: x, + ) + files['vendordata'] = ( + self._path_join("openstack", version, 'vendor_data.json'), + False, + util.load_json, + ) + return files + + version = self._find_working_version(version) + results = { + 'userdata': '', + 'version': 2, + } + data = datafiles(version) + for (name, (path, required, translator)) in data.iteritems(): + path = self._path_join(self.base_path, path) + data = None + found = False + if self._path_exists(path): + try: + data = self._path_read(path) + except IOError: + raise NonReadable("Failed to read: %s" % path) + found = True + else: + if required: + raise NonReadable("Missing mandatory path: %s" % path) + if found and translator: + try: + data = translator(data) + except Exception as e: + raise BrokenMetadata("Failed to process " + "path %s: %s" % (path, e)) + if found: + results[name] = data + + metadata = results['metadata'] + if 'random_seed' in metadata: + random_seed = metadata['random_seed'] + try: + metadata['random_seed'] = base64.b64decode(random_seed) + except (ValueError, TypeError) as e: + raise BrokenMetadata("Badly formatted metadata" + " random_seed entry: %s" % e) + + # load any files that were provided + files = {} + metadata_files = metadata.get('files', []) + for item in metadata_files: + if 'path' not in item: + continue + path = item['path'] + try: + files[path] = self._read_content_path(item) + except Exception as e: + raise BrokenMetadata("Failed to read provided " + "file %s: %s" % (path, e)) + results['files'] = files + + # The 'network_config' item in metadata is a content pointer + # to the network config that should be applied. It is just a + # ubuntu/debian '/etc/network/interfaces' file. + net_item = metadata.get("network_config", None) + if net_item: + try: + results['network_config'] = self._read_content_path(net_item) + except IOError as e: + raise BrokenMetadata("Failed to read network" + " configuration: %s" % (e)) + + # To openstack, user can specify meta ('nova boot --meta=key=value') + # and those will appear under metadata['meta']. + # if they specify 'dsmode' they're indicating the mode that they intend + # for this datasource to operate in. + try: + results['dsmode'] = metadata['meta']['dsmode'] + except KeyError: + pass + + # Read any ec2-metadata (if applicable) + results['ec2-metadata'] = self._read_ec2_metadata() + + # Perform some misc. metadata key renames... + for (target_key, source_key, is_required) in KEY_COPIES: + if is_required and source_key not in metadata: + raise BrokenMetadata("No '%s' entry in metadata" % source_key) + if source_key in metadata: + metadata[target_key] = metadata.get(source_key) + return results + + +class ConfigDriveReader(BaseReader): + def __init__(self, base_path): + super(ConfigDriveReader, self).__init__(base_path) + + def _path_join(self, base, *add_ons): + components = [base] + list(add_ons) + return os.path.join(*components) + + def _path_exists(self, path): + return os.path.exists(path) + + def _path_read(self, path): + return util.load_file(path) + + def _read_ec2_metadata(self): + path = self._path_join(self.base_path, + 'ec2', 'latest', 'meta-data.json') + if not self._path_exists(path): + return {} + else: + try: + return util.load_json(self._path_read(path)) + except Exception as e: + raise BrokenMetadata("Failed to process " + "path %s: %s" % (path, e)) + + def read_v1(self): + """Reads a version 1 formatted location. + + Return a dict with metadata, userdata, dsmode, files and version (1). + + If not a valid path, raise a NonReadable exception. + """ + + found = {} + for name in FILES_V1.keys(): + path = self._path_join(self.base_path, name) + if self._path_exists(path): + found[name] = path + if len(found) == 0: + raise NonReadable("%s: no files found" % (self.base_path)) + + md = {} + for (name, (key, translator, default)) in FILES_V1.iteritems(): + if name in found: + path = found[name] + try: + contents = self._path_read(path) + except IOError: + raise BrokenMetadata("Failed to read: %s" % path) + try: + md[key] = translator(contents) + except Exception as e: + raise BrokenMetadata("Failed to process " + "path %s: %s" % (path, e)) + else: + md[key] = copy.deepcopy(default) + + keydata = md['authorized_keys'] + meta_js = md['meta_js'] + + # keydata in meta_js is preferred over "injected" + keydata = meta_js.get('public-keys', keydata) + if keydata: + lines = keydata.splitlines() + md['public-keys'] = [l for l in lines + if len(l) and not l.startswith("#")] + + # config-drive-v1 has no way for openstack to provide the instance-id + # so we copy that into metadata from the user input + if 'instance-id' in meta_js: + md['instance-id'] = meta_js['instance-id'] + + results = { + 'version': 1, + 'metadata': md, + } + + # allow the user to specify 'dsmode' in a meta tag + if 'dsmode' in meta_js: + results['dsmode'] = meta_js['dsmode'] + + # config-drive-v1 has no way of specifying user-data, so the user has + # to cheat and stuff it in a meta tag also. + results['userdata'] = meta_js.get('user-data', '') + + # this implementation does not support files other than + # network/interfaces and authorized_keys... + results['files'] = {} + + return results + + +class MetadataReader(BaseReader): + def __init__(self, base_url, ssl_details=None, timeout=5, retries=5): + super(MetadataReader, self).__init__(base_url) + self._url_reader = functools.partial(url_helper.readurl, + retries=retries, + ssl_details=ssl_details, + timeout=timeout) + self._url_checker = functools.partial(url_helper.existsurl, + ssl_details=ssl_details, + timeout=timeout) + self._ec2_reader = functools.partial(ec2_utils.get_instance_metadata, + ssl_details=ssl_details, + timeout=timeout, + retries=retries) + + def _path_read(self, path): + return str(self._url_reader(path)) + + def _path_exists(self, path): + return self._url_checker(path) + + def _path_join(self, base, *add_ons): + return url_helper.combine_url(base, *add_ons) + + def _read_ec2_metadata(self): + return self._ec2_reader() diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py new file mode 100644 index 00000000..7d93f1d3 --- /dev/null +++ b/tests/unittests/test_datasource/test_openstack.py @@ -0,0 +1,122 @@ +import re +import json + +from StringIO import StringIO + +from urlparse import urlparse + +from tests.unittests import helpers + +from cloudinit.sources import DataSourceOpenStack as ds +from cloudinit.sources.helpers import openstack +from cloudinit import util + +import httpretty as hp + +BASE_URL = "http://169.254.169.254" +PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' +EC2_META = { + 'ami-id': 'ami-00000001', + 'ami-launch-index': 0, + 'ami-manifest-path': 'FIXME', + 'hostname': 'sm-foo-test.novalocal', + 'instance-action': 'none', + 'instance-id': 'i-00000001', + 'instance-type': 'm1.tiny', + 'local-hostname': 'sm-foo-test.novalocal', + 'local-ipv4': '0.0.0.0', + 'public-hostname': 'sm-foo-test.novalocal', + 'public-ipv4': '0.0.0.1', + 'reservation-id': 'r-iru5qm4m', +} +USER_DATA = '#!/bin/sh\necho This is user data\n' +VENDOR_DATA = { + 'magic': '', +} +OSTACK_META = { + 'availability_zone': 'nova', + 'files': [{'content_path': '/content/0000', 'path': '/etc/foo.cfg'}, + {'content_path': '/content/0001', 'path': '/etc/bar/bar.cfg'}], + 'hostname': 'sm-foo-test.novalocal', + 'meta': {'dsmode': 'local', 'my-meta': 'my-value'}, + 'name': 'sm-foo-test', + 'public_keys': {'mykey': PUBKEY}, + 'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c', +} +CONTENT_0 = 'This is contents of /etc/foo.cfg\n' +CONTENT_1 = '# this is /etc/bar/bar.cfg\n' +OS_FILES = { + 'openstack/2012-08-10/meta_data.json': json.dumps(OSTACK_META), + 'openstack/2012-08-10/user_data': USER_DATA, + 'openstack/content/0000': CONTENT_0, + 'openstack/content/0001': CONTENT_1, + 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), + 'openstack/latest/user_data': USER_DATA, + 'openstack/latest/vendor_data.json': json.dumps(VENDOR_DATA), +} +EC2_FILES = { + 'latest/user-data': USER_DATA, +} + + +def _register_uris(version): + + def match_ec2_url(uri, headers): + path = uri.path.lstrip("/") + if path in EC2_FILES: + return (200, headers, EC2_FILES.get(path)) + if path == 'latest/meta-data': + buf = StringIO() + for (k, v) in EC2_META.items(): + if isinstance(v, (list, tuple)): + buf.write("%s/" % (k)) + else: + buf.write("%s" % (k)) + buf.write("\n") + return (200, headers, buf.getvalue()) + if path.startswith('latest/meta-data'): + value = None + pieces = path.split("/") + if path.endswith("/"): + pieces = pieces[2:-1] + value = util.get_cfg_by_path(EC2_META, pieces) + else: + pieces = pieces[2:] + value = util.get_cfg_by_path(EC2_META, pieces) + if value is not None: + return (200, headers, str(value)) + return (404, headers, '') + + def get_request_callback(method, uri, headers): + uri = urlparse(uri) + path = uri.path.lstrip("/") + if path in OS_FILES: + return (200, headers, OS_FILES.get(path)) + return match_ec2_url(uri, headers) + + def head_request_callback(method, uri, headers): + uri = urlparse(uri) + path = uri.path.lstrip("/") + for key in OS_FILES.keys(): + if key.startswith(path): + return (200, headers, '') + return (404, headers, '') + + hp.register_uri(hp.GET, re.compile(r'http://169.254.169.254/.*'), + body=get_request_callback) + + hp.register_uri(hp.HEAD, re.compile(r'http://169.254.169.254/.*'), + body=head_request_callback) + + +class TestOpenStackDataSource(helpers.TestCase): + VERSION = 'latest' + + @hp.activate + def test_fetch(self): + _register_uris(self.VERSION) + f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + self.assertEquals(VENDOR_DATA, f.get('vendordata')) + self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertEquals(USER_DATA, f.get('userdata')) -- cgit v1.2.3 From 29b838af062549ef5e2f12c3df74721d294dea37 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Mon, 3 Feb 2014 11:28:50 +0000 Subject: Add Google Compute Engine data source support. --- cloudinit/settings.py | 1 + cloudinit/sources/DataSourceGCE.py | 100 +++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 cloudinit/sources/DataSourceGCE.py (limited to 'cloudinit/sources') diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 7be2199a..7c598bf9 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -36,6 +36,7 @@ CFG_BUILTIN = { 'AltCloud', 'OVF', 'MAAS', + 'GCE', 'Ec2', 'CloudStack', 'SmartOS', diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py new file mode 100644 index 00000000..b6a82245 --- /dev/null +++ b/cloudinit/sources/DataSourceGCE.py @@ -0,0 +1,100 @@ +# vi: ts=4 expandtab +# +# Author: Vaidas Jablonskis +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import requests + +from cloudinit import log as logging +from cloudinit import sources + +LOG = logging.getLogger(__name__) + +MD_URL = 'http://metadata/computeMetadata/v1/' + + +class DataSourceGCE(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.metadata_address = MD_URL + self.metadata = {} + + # GCE takes sshKeys attribute in the format of ':' + # so we have to trim each key to remove the username part + def _trim_key(self, public_key): + try: + index = public_key.index(':') + if index > 0: + return public_key[(index + 1):] + except: + return public_key + + def get_data(self): + # GCE metadata server requires a custom header since v1 + headers = {'X-Google-Metadata-Request': True} + + url_map = { + 'instance-id': self.metadata_address + 'instance/id', + 'availability-zone': self.metadata_address + 'instance/zone', + 'public-keys': self.metadata_address + 'project/attributes/sshKeys', + 'local-hostname': self.metadata_address + 'instance/hostname', + } + + with requests.Session() as s: + for mkey in url_map.iterkeys(): + try: + r = s.get(url_map[mkey], headers=headers) + except requests.exceptions.ConnectionError: + return False + if r.ok: + if mkey == 'public-keys': + pub_keys = [self._trim_key(k) for k in r.text.splitlines()] + self.metadata[mkey] = pub_keys + else: + self.metadata[mkey] = r.text + else: + self.metadata[mkey] = None + return False + return True + + @property + def launch_index(self): + # GCE does not provide lauch_index property + return None + + def get_instance_id(self): + return self.metadata['instance-id'] + + def get_public_ssh_keys(self): + return self.metadata['public-keys'] + + def get_hostname(self, fqdn=False): + return self.metadata['local-hostname'] + + def get_userdata_raw(self): + return None + + @property + def availability_zone(self): + return self.metadata['instance-zone'] + +# Used to match classes to dependencies +datasources = [ + (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) -- cgit v1.2.3 From e1710e06fd09c3f643fdca851c011da3a3eb7a98 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Fri, 7 Feb 2014 09:43:32 +0000 Subject: use url_helper instead of requests --- cloudinit/sources/DataSourceGCE.py | 38 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index b6a82245..f9cdc1da 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -14,10 +14,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import requests from cloudinit import log as logging from cloudinit import sources +from cloudinit import url_helper LOG = logging.getLogger(__name__) @@ -30,16 +30,6 @@ class DataSourceGCE(sources.DataSource): self.metadata_address = MD_URL self.metadata = {} - # GCE takes sshKeys attribute in the format of ':' - # so we have to trim each key to remove the username part - def _trim_key(self, public_key): - try: - index = public_key.index(':') - if index > 0: - return public_key[(index + 1):] - except: - return public_key - def get_data(self): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} @@ -51,22 +41,18 @@ class DataSourceGCE(sources.DataSource): 'local-hostname': self.metadata_address + 'instance/hostname', } - with requests.Session() as s: - for mkey in url_map.iterkeys(): - try: - r = s.get(url_map[mkey], headers=headers) - except requests.exceptions.ConnectionError: - return False - if r.ok: - if mkey == 'public-keys': - pub_keys = [self._trim_key(k) for k in r.text.splitlines()] - self.metadata[mkey] = pub_keys - else: - self.metadata[mkey] = r.text + for mkey in url_map.iterkeys(): + resp = url_helper.readurl(url=url_map[mkey], headers=headers) + if resp.ok(): + if mkey == 'public-keys': + pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] + self.metadata[mkey] = pub_keys else: - self.metadata[mkey] = None - return False - return True + self.metadata[mkey] = resp.contents + else: + self.metadata[mkey] = None + return False + return True @property def launch_index(self): -- cgit v1.2.3 From 64b35a9c3b201aed5073823fd4a15ecb15195aea Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Fri, 7 Feb 2014 13:12:48 +0000 Subject: Forgot to include _trim_key function Got removed somehow --- cloudinit/sources/DataSourceGCE.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index f9cdc1da..8a46f933 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -30,6 +30,16 @@ class DataSourceGCE(sources.DataSource): self.metadata_address = MD_URL self.metadata = {} + # GCE takes sshKeys attribute in the format of ':' + # so we have to trim each key to remove the username part + def _trim_key(self, public_key): + try: + index = public_key.index(':') + if index > 0: + return public_key[(index + 1):] + except: + return public_key + def get_data(self): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} -- cgit v1.2.3 From f9582464b157ffb0087b18910af11aa6ed49abcd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 7 Feb 2014 16:34:04 -0800 Subject: Add a bunch of new tests --- cloudinit/sources/DataSourceOpenStack.py | 3 +- tests/unittests/helpers.py | 6 + tests/unittests/test_datasource/test_openstack.py | 197 ++++++++++++++++++++-- 3 files changed, 190 insertions(+), 16 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 44889f4e..621572de 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -121,7 +121,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'Crawl of openstack metadata service', read_metadata_service, args=[self.metadata_address], - kwargs={'ssl_details': self.ssl_details}) + kwargs={'ssl_details': self.ssl_details, + 'version': openstack.OS_LATEST}) except openstack.NonReadable: return False except openstack.BrokenMetadata: diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 5b4f4208..945c1500 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -29,6 +29,12 @@ if (_PY_MAJOR, _PY_MINOR) <= (2, 6): standardMsg = standardMsg % (member, container) self.fail(self._formatMessage(msg, standardMsg)) + def assertIsNone(self, value, msg=None): + if value is not None: + standardMsg = '%r is not None' + standardMsg = standardMsg % (value) + self.fail(self._formatMessage(msg, standardMsg)) + else: class TestCase(unittest.TestCase): pass diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 7d93f1d3..c8cc40db 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -1,12 +1,33 @@ -import re +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import copy import json +import re from StringIO import StringIO from urlparse import urlparse -from tests.unittests import helpers +from tests.unittests import helpers as test_helpers +from cloudinit import helpers +from cloudinit import settings from cloudinit.sources import DataSourceOpenStack as ds from cloudinit.sources.helpers import openstack from cloudinit import util @@ -17,7 +38,7 @@ BASE_URL = "http://169.254.169.254" PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' EC2_META = { 'ami-id': 'ami-00000001', - 'ami-launch-index': 0, + 'ami-launch-index': '0', 'ami-manifest-path': 'FIXME', 'hostname': 'sm-foo-test.novalocal', 'instance-action': 'none', @@ -59,15 +80,17 @@ EC2_FILES = { } -def _register_uris(version): +def _register_uris(version, ec2_files, ec2_meta, os_files): + """Registers a set of url patterns into httpretty that will mimic the + same data returned by the openstack metadata service (and ec2 service).""" def match_ec2_url(uri, headers): path = uri.path.lstrip("/") - if path in EC2_FILES: - return (200, headers, EC2_FILES.get(path)) + if path in ec2_files: + return (200, headers, ec2_files.get(path)) if path == 'latest/meta-data': buf = StringIO() - for (k, v) in EC2_META.items(): + for (k, v) in ec2_meta.items(): if isinstance(v, (list, tuple)): buf.write("%s/" % (k)) else: @@ -79,10 +102,10 @@ def _register_uris(version): pieces = path.split("/") if path.endswith("/"): pieces = pieces[2:-1] - value = util.get_cfg_by_path(EC2_META, pieces) + value = util.get_cfg_by_path(ec2_meta, pieces) else: pieces = pieces[2:] - value = util.get_cfg_by_path(EC2_META, pieces) + value = util.get_cfg_by_path(ec2_meta, pieces) if value is not None: return (200, headers, str(value)) return (404, headers, '') @@ -90,14 +113,14 @@ def _register_uris(version): def get_request_callback(method, uri, headers): uri = urlparse(uri) path = uri.path.lstrip("/") - if path in OS_FILES: - return (200, headers, OS_FILES.get(path)) + if path in os_files: + return (200, headers, os_files.get(path)) return match_ec2_url(uri, headers) def head_request_callback(method, uri, headers): uri = urlparse(uri) path = uri.path.lstrip("/") - for key in OS_FILES.keys(): + for key in os_files.keys(): if key.startswith(path): return (200, headers, '') return (404, headers, '') @@ -109,14 +132,158 @@ def _register_uris(version): body=head_request_callback) -class TestOpenStackDataSource(helpers.TestCase): +class TestOpenStackDataSource(test_helpers.TestCase): VERSION = 'latest' @hp.activate - def test_fetch(self): - _register_uris(self.VERSION) + def test_successful(self): + _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) + f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + self.assertEquals(VENDOR_DATA, f.get('vendordata')) + self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertEquals(2, len(f['files'])) + self.assertEquals(USER_DATA, f.get('userdata')) + self.assertEquals(EC2_META, f.get('ec2-metadata')) + self.assertEquals(2, f.get('version')) + metadata = f['metadata'] + self.assertEquals('nova', metadata.get('availability_zone')) + self.assertEquals('sm-foo-test.novalocal', metadata.get('hostname')) + self.assertEquals('sm-foo-test.novalocal', + metadata.get('local-hostname')) + self.assertEquals('sm-foo-test', metadata.get('name')) + self.assertEquals('b0fa911b-69d4-4476-bbe2-1c92bff6535c', + metadata.get('uuid')) + self.assertEquals('b0fa911b-69d4-4476-bbe2-1c92bff6535c', + metadata.get('instance-id')) + + @hp.activate + def test_no_ec2(self): + _register_uris(self.VERSION, {}, {}, OS_FILES) f = ds.read_metadata_service(BASE_URL, version=self.VERSION) self.assertEquals(VENDOR_DATA, f.get('vendordata')) self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertEquals(USER_DATA, f.get('userdata')) + self.assertEquals({}, f.get('ec2-metadata')) + self.assertEquals(2, f.get('version')) + + @hp.activate + def test_bad_metadata(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files.pop(k, None) + _register_uris(self.VERSION, {}, {}, os_files) + self.assertRaises(openstack.NonReadable, ds.read_metadata_service, + BASE_URL, version=self.VERSION) + + @hp.activate + def test_bad_uuid(self): + os_files = copy.deepcopy(OS_FILES) + os_meta = copy.deepcopy(OSTACK_META) + os_meta.pop('uuid') + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files[k] = json.dumps(os_meta) + _register_uris(self.VERSION, {}, {}, os_files) + self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, + BASE_URL, version=self.VERSION) + + @hp.activate + def test_userdata_empty(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('user_data'): + os_files.pop(k, None) + _register_uris(self.VERSION, {}, {}, os_files) + f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + self.assertEquals(VENDOR_DATA, f.get('vendordata')) + self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertFalse(f.get('userdata')) + + @hp.activate + def test_vendordata_empty(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('vendor_data.json'): + os_files.pop(k, None) + _register_uris(self.VERSION, {}, {}, os_files) + f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertFalse(f.get('vendordata')) + + @hp.activate + def test_vendordata_invalid(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('vendor_data.json'): + os_files[k] = '{' # some invalid json + _register_uris(self.VERSION, {}, {}, os_files) + self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, + BASE_URL, version=self.VERSION) + + @hp.activate + def test_metadata_invalid(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files[k] = '{' # some invalid json + _register_uris(self.VERSION, {}, {}, os_files) + self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, + BASE_URL, version=self.VERSION) + + @hp.activate + def test_datasource(self): + _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) + ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + self.assertIsNone(ds_os.version) + found = ds_os.get_data() + self.assertTrue(found) + self.assertEquals(2, ds_os.version) + md = dict(ds_os.metadata) + md.pop('instance-id', None) + md.pop('local-hostname', None) + self.assertEquals(OSTACK_META, md) + self.assertEquals(EC2_META, ds_os.ec2_metadata) + self.assertEquals(USER_DATA, ds_os.userdata_raw) + self.assertEquals(2, len(ds_os.files)) + self.assertEquals(VENDOR_DATA, ds_os.vendordata_raw) + + @hp.activate + def test_bad_datasource_meta(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files[k] = '{' # some invalid json + _register_uris(self.VERSION, {}, {}, os_files) + ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + self.assertIsNone(ds_os.version) + found = ds_os.get_data() + self.assertFalse(found) + self.assertIsNone(ds_os.version) + + @hp.activate + def test_no_datasource(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files.pop(k) + _register_uris(self.VERSION, {}, {}, os_files) + ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + ds_os.ds_cfg = { + 'max_wait': 0, + 'timeout': 0, + } + self.assertIsNone(ds_os.version) + found = ds_os.get_data() + self.assertFalse(found) + self.assertIsNone(ds_os.version) -- cgit v1.2.3 From 098a74e6207f5d91f515fac63e970375d52795c0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 8 Feb 2014 12:20:33 -0800 Subject: Remove HEAD usage and other small adjustments --- cloudinit/ec2_utils.py | 4 +-- cloudinit/sources/DataSourceOpenStack.py | 1 + cloudinit/sources/helpers/openstack.py | 41 ++++++++++++++--------- cloudinit/url_helper.py | 23 ++----------- tests/unittests/test_datasource/test_openstack.py | 11 ------ 5 files changed, 30 insertions(+), 50 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 91cba20f..a7c9c9ab 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -16,10 +16,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import httplib -from urlparse import (urlparse, urlunparse) - import functools +import httplib import json from cloudinit import log as logging diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 621572de..2c50ed84 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -44,6 +44,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.ssl_details = util.fetch_ssl_details(self.paths) self.version = None self.files = {} + self.ec2_metadata = None def __str__(self): root = sources.DataSource.__str__(self) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 9dbef677..09fb4ad8 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -21,7 +21,6 @@ import abc import base64 import copy -import functools import os from cloudinit import ec2_utils @@ -395,26 +394,38 @@ class ConfigDriveReader(BaseReader): class MetadataReader(BaseReader): def __init__(self, base_url, ssl_details=None, timeout=5, retries=5): super(MetadataReader, self).__init__(base_url) - self._url_reader = functools.partial(url_helper.readurl, - retries=retries, - ssl_details=ssl_details, - timeout=timeout) - self._url_checker = functools.partial(url_helper.existsurl, - ssl_details=ssl_details, - timeout=timeout) - self._ec2_reader = functools.partial(ec2_utils.get_instance_metadata, - ssl_details=ssl_details, - timeout=timeout, - retries=retries) + self.ssl_details = ssl_details + self.timeout = float(timeout) + self.retries = int(retries) def _path_read(self, path): - return str(self._url_reader(path)) + response = url_helper.readurl(path, + retries=self.retries, + ssl_details=self.ssl_details, + timeout=self.timeout) + return response.contents def _path_exists(self, path): - return self._url_checker(path) + + def should_retry_cb(request, cause): + if cause.code >= 400: + return False + return True + + try: + response = url_helper.readurl(path, + retries=self.retries, + ssl_details=self.ssl_details, + timeout=self.timeout, + exception_cb=should_retry_cb) + return response.ok() + except IOError: + return False def _path_join(self, base, *add_ons): return url_helper.combine_url(base, *add_ons) def _read_ec2_metadata(self): - return self._ec2_reader() + return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details, + timeout=self.timeout, + retries=self.retries) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 76a8e29b..a477b185 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -166,35 +166,16 @@ def _get_ssl_args(url, ssl_details): return ssl_args -def existsurl(url, ssl_details=None, timeout=None): - r = _readurl(url, ssl_details=ssl_details, timeout=timeout, - method='HEAD', check_status=False) - return r.ok() - - def readurl(url, data=None, timeout=None, retries=0, sec_between=1, - headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True, exception_cb=None): - return _readurl(url, data=data, timeout=timeout, retries=retries, - sec_between=sec_between, headers=headers, - headers_cb=headers_cb, ssl_details=ssl_details, - check_status=check_status, - allow_redirects=allow_redirects, - exception_cb=exception_cb) - - -def _readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True, exception_cb=None, - method='GET'): + check_status=True, allow_redirects=True, exception_cb=None): url = _cleanurl(url) req_args = { 'url': url, } req_args.update(_get_ssl_args(url, ssl_details)) - scheme = urlparse(url).scheme # pylint: disable=E1101 req_args['allow_redirects'] = allow_redirects - req_args['method'] = method + req_args['method'] = 'GET' if timeout is not None: req_args['timeout'] = max(float(timeout), 0) if data: diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 3fcf8bc9..3a64430a 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -117,20 +117,9 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): return (200, headers, os_files.get(path)) return match_ec2_url(uri, headers) - def head_request_callback(method, uri, headers): - uri = urlparse(uri) - path = uri.path.lstrip("/") - for key in os_files.keys(): - if key.startswith(path): - return (200, headers, '') - return (404, headers, '') - hp.register_uri(hp.GET, re.compile(r'http://169.254.169.254/.*'), body=get_request_callback) - hp.register_uri(hp.HEAD, re.compile(r'http://169.254.169.254/.*'), - body=head_request_callback) - class TestOpenStackDataSource(test_helpers.TestCase): VERSION = 'latest' -- cgit v1.2.3 From aa1d0328da6ae1ace03e327feafabcf84f91b2b5 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Sat, 8 Feb 2014 20:59:52 +0000 Subject: wrap url get call in try/except clause --- cloudinit/sources/DataSourceGCE.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 8a46f933..c96cfffd 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -52,7 +52,10 @@ class DataSourceGCE(sources.DataSource): } for mkey in url_map.iterkeys(): - resp = url_helper.readurl(url=url_map[mkey], headers=headers) + try: + resp = url_helper.readurl(url=url_map[mkey], headers=headers) + except IOError: + return False if resp.ok(): if mkey == 'public-keys': pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] -- cgit v1.2.3 From ea05883f88cedd5ed099aed19d4c5df563525097 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 8 Feb 2014 13:16:07 -0800 Subject: Capture IOError and use LOG better --- cloudinit/sources/DataSourceConfigDrive.py | 2 +- cloudinit/sources/DataSourceOpenStack.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 1d30fe08..142e0eb8 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -87,7 +87,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): md = util.mergemanydict([md, DEFAULT_METADATA]) user_dsmode = results.get('dsmode', None) if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("User specified invalid mode: %s" % user_dsmode) + LOG.warn("User specified invalid mode: %s", user_dsmode) user_dsmode = None dsmode = get_ds_mode(cfgdrv_ver=results['version'], diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 2c50ed84..69807798 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -126,14 +126,14 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'version': openstack.OS_LATEST}) except openstack.NonReadable: return False - except openstack.BrokenMetadata: + except (openstack.BrokenMetadata, IOError): util.logexc(LOG, "Broken metadata address %s", self.metadata_address) return False user_dsmode = results.get('dsmode', None) if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("User specified invalid mode: %s" % user_dsmode) + LOG.warn("User specified invalid mode: %s", user_dsmode) user_dsmode = None if user_dsmode == 'disabled': return False -- cgit v1.2.3 From 5788cd903f6e4a9bab2ad32e9c1d2eb13b485ac3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 8 Feb 2014 17:29:52 -0800 Subject: Handle code not existing in older requests versions --- cloudinit/sources/helpers/openstack.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 09fb4ad8..a17148d3 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -408,8 +408,13 @@ class MetadataReader(BaseReader): def _path_exists(self, path): def should_retry_cb(request, cause): - if cause.code >= 400: - return False + try: + code = int(cause.code) + if code >= 400: + return False + except (TypeError, ValueError): + # Older versions of requests didn't have a code. + pass return True try: -- cgit v1.2.3 From 065287bd56fe7f63a8dc41fa1457be4439f20efd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Feb 2014 19:58:38 -0500 Subject: support configuration of MD_URL, disable if not resolvable. this allows the metadata url to be configured by setting: datasource: GCE: metadata_url: Then also, if its not resolvable, we just deactivate the datasource quickly. --- cloudinit/sources/DataSourceGCE.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index c96cfffd..6eb3da4d 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -16,19 +16,25 @@ from cloudinit import log as logging +from cloudinit import util from cloudinit import sources from cloudinit import url_helper LOG = logging.getLogger(__name__) -MD_URL = 'http://metadata/computeMetadata/v1/' +BUILTIN_DS_CONFIG = { + 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' +} class DataSourceGCE(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.metadata_address = MD_URL self.metadata = {} + self.ds_cfg = util.mergemanydict([ + util.get_cfg_by_path(sys_cfg, ["datasource", "GCE"], {}), + BUILTIN_DS_CONFIG]) + self.metadata_address = self.ds_cfg['metadata_url'] # GCE takes sshKeys attribute in the format of ':' # so we have to trim each key to remove the username part @@ -51,6 +57,11 @@ class DataSourceGCE(sources.DataSource): 'local-hostname': self.metadata_address + 'instance/hostname', } + # if we cannot resolve the metadata server, then no point in trying + if not util.is_resolvable(self.metadata_address): + LOG.debug("%s is not resolvable", self.metadata_address) + return False + for mkey in url_map.iterkeys(): try: resp = url_helper.readurl(url=url_map[mkey], headers=headers) -- cgit v1.2.3 From 18b8d40d96d2e19f1a949b06b55a3dac54595b23 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 10:39:39 -0500 Subject: cloudsigma: change default dsmode to 'net' Previously this had 'local' as the default datasource mode, meaning that user-data code such as boot hooks and such would not be guaranteed to have network access. That would be out of sync with the expectation on other platforms where the default is 'network up'. The user can still specify 'dsmode' as local if necessary and the local datasource will claim itself found. --- cloudinit/sources/DataSourceCloudSigma.py | 27 ++++++++++++++++++++------- doc/sources/cloudsigma/README.rst | 6 +++--- 2 files changed, 23 insertions(+), 10 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index 78acd8a4..e734d7e5 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -45,18 +45,25 @@ class DataSourceCloudSigma(sources.DataSource): Metadata is the whole server context and /meta/cloud-config is used as userdata. """ + dsmode = None try: server_context = self.cepko.all().result server_meta = server_context['meta'] - self.userdata_raw = server_meta.get('cloudinit-user-data', "") - self.metadata = server_context - self.ssh_public_key = server_meta['ssh_public_key'] - - if server_meta.get('cloudinit-dsmode') in VALID_DSMODES: - self.dsmode = server_meta['cloudinit-dsmode'] except: util.logexc(LOG, "Failed reading from the serial port") return False + + dsmode = server_meta.get('cloudinit-dsmode', self.dsmode) + if dsmode not in VALID_DSMODES: + LOG.warn("Invalid dsmode %s, assuming default of 'net'", dsmode) + dsmode = 'net' + if dsmode == "disabled" or dsmode != self.dsmode: + return False + + self.userdata_raw = server_meta.get('cloudinit-user-data', "") + self.metadata = server_context + self.ssh_public_key = server_meta['ssh_public_key'] + return True def get_hostname(self, fqdn=False, resolve_ip=False): @@ -76,11 +83,17 @@ class DataSourceCloudSigma(sources.DataSource): return self.metadata['uuid'] +class DataSourceCloudSigmaNet(DataSourceCloudSigma): + def __init__(self, sys_cfg, distro, paths): + DataSourceCloudSigma.__init__(self, sys_cfg, distro, paths) + self.dsmode = 'net' + + # Used to match classes to dependencies. Since this datasource uses the serial # port network is not really required, so it's okay to load without it, too. datasources = [ (DataSourceCloudSigma, (sources.DEP_FILESYSTEM)), - (DataSourceCloudSigma, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + (DataSourceCloudSigmaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/doc/sources/cloudsigma/README.rst b/doc/sources/cloudsigma/README.rst index 8cb2b0fe..1d9160a2 100644 --- a/doc/sources/cloudsigma/README.rst +++ b/doc/sources/cloudsigma/README.rst @@ -23,9 +23,9 @@ You can provide user-data to the VM using the dedicated `meta field`_ in the `se header could be omitted. However since this is a raw-text field you could provide any of the valid `config formats`_. -If your user-data needs an internet connection you have to create a `meta field`_ in the `server context`_ -``cloudinit-dsmode`` and set "net" as value. If this field does not exist the default value is "local". - +If your user-data does not need an internet connection you can create a +`meta field`_ in the `server context`_ ``cloudinit-dsmode`` and set "local" as value. +If this field does not exist the default value is "net". .. _CloudSigma: http://cloudsigma.com/ -- cgit v1.2.3 From 64856d7f245cc329a11b263c4eef8bf76bdee0e6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 10:58:08 -0500 Subject: add 'user-data' support. This just adds user-data in 'instance/attributes/user-data'. Also turns retries to 0 on all other things. --- cloudinit/sources/DataSourceGCE.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 6eb3da4d..95a410ba 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -25,6 +25,7 @@ LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' } +REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') class DataSourceGCE(sources.DataSource): @@ -55,6 +56,7 @@ class DataSourceGCE(sources.DataSource): 'availability-zone': self.metadata_address + 'instance/zone', 'public-keys': self.metadata_address + 'project/attributes/sshKeys', 'local-hostname': self.metadata_address + 'instance/hostname', + 'user-data': self.metadata_address + 'instance/attributes/user-data', } # if we cannot resolve the metadata server, then no point in trying @@ -64,7 +66,8 @@ class DataSourceGCE(sources.DataSource): for mkey in url_map.iterkeys(): try: - resp = url_helper.readurl(url=url_map[mkey], headers=headers) + resp = url_helper.readurl(url=url_map[mkey], headers=headers, + retries=0) except IOError: return False if resp.ok(): @@ -74,8 +77,15 @@ class DataSourceGCE(sources.DataSource): else: self.metadata[mkey] = resp.contents else: + if mkey in REQUIRED_FIELDS: + LOG.warn("required metadata '%s' not found in metadata", + url_map[mkey]) + return False + self.metadata[mkey] = None return False + + self.user_data_raw = self.metadata['user-data'] return True @property @@ -92,9 +102,6 @@ class DataSourceGCE(sources.DataSource): def get_hostname(self, fqdn=False): return self.metadata['local-hostname'] - def get_userdata_raw(self): - return None - @property def availability_zone(self): return self.metadata['instance-zone'] -- cgit v1.2.3 From 87d0fa867f27f101e93006ba8dc8a395098e8df1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 12:13:42 -0500 Subject: wait less for the metadata service (by default) Waiting around for a metadata service in a given datasource means that if its not there all the subsequent datasources have to wait, and boot is slowed down. As it is right now, EC2 is the only one that has the right to wait. In the past, we had to wait around for the EC2 metadata service. I really do not want to extend that courtesy to other cloud platforms. A network based metadata service should be up as soon as networking is up. --- cloudinit/sources/DataSourceOpenStack.py | 28 +++++++++++++--------------- cloudinit/url_helper.py | 1 + 2 files changed, 14 insertions(+), 15 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 69807798..7fafa3f7 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -45,6 +45,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.version = None self.files = {} self.ec2_metadata = None + if not self.ds_cfg: + self.ds_cfg = {} def __str__(self): root = sources.DataSource.__str__(self) @@ -54,27 +56,25 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): def _get_url_settings(self): # TODO(harlowja): this is shared with ec2 datasource, we should just # move it to a shared location instead... - ds_cfg = self.ds_cfg - if not ds_cfg: - ds_cfg = {} - max_wait = 120 + # Note: the defaults here are different though. + + # max_wait < 0 indicates do not wait + max_wait = -1 + timeout = 10 + try: - max_wait = int(ds_cfg.get("max_wait", max_wait)) + max_wait = int(self.ds_cfg.get("max_wait", max_wait)) except Exception: util.logexc(LOG, "Failed to get max wait. using %s", max_wait) - timeout = 50 try: - timeout = max(0, int(ds_cfg.get("timeout", timeout))) + timeout = max(0, int(self.ds_cfg.get("timeout", timeout))) except Exception: util.logexc(LOG, "Failed to get timeout, using %s", timeout) return (max_wait, timeout) def wait_for_metadata_service(self): - ds_cfg = self.ds_cfg - if not ds_cfg: - ds_cfg = {} - urls = ds_cfg.get("metadata_urls", [DEF_MD_URL]) + urls = self.ds_cfg.get("metadata_urls", [DEF_MD_URL]) filtered = [x for x in urls if util.is_resolvable_url(x)] if set(filtered) != set(urls): LOG.debug("Removed the following from metadata urls: %s", @@ -95,8 +95,6 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): url2base[md_url] = url (max_wait, timeout) = self._get_url_settings() - if max_wait <= 0: - return False start_time = time.time() avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait, timeout=timeout, @@ -104,8 +102,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): if avail_url: LOG.debug("Using metadata source: '%s'", url2base[avail_url]) else: - LOG.critical("Giving up on md from %s after %s seconds", - md_urls, int(time.time() - start_time)) + LOG.debug("Giving up on OpenStack md from %s after %s seconds", + md_urls, int(time.time() - start_time)) self.metadata_address = url2base.get(avail_url) return bool(avail_url) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index c116a484..4a83169a 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -266,6 +266,7 @@ def wait_for_url(urls, max_wait=None, timeout=None, max_wait: roughly the maximum time to wait before giving up The max time is *actually* len(urls)*timeout as each url will be tried once and given the timeout provided. + a number <= 0 will always result in only one try timeout: the timeout provided to urlopen status_cb: call method with string message when a url is not available headers_cb: call method with single argument of url to get headers -- cgit v1.2.3 From 5d5b8f90a10549bf54cf2000f514eb9c6ea420f7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 12:50:59 -0500 Subject: do not warn on waiting for url --- cloudinit/sources/DataSourceOpenStack.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 7fafa3f7..5edbb448 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -97,8 +97,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): (max_wait, timeout) = self._get_url_settings() start_time = time.time() avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait, - timeout=timeout, - status_cb=LOG.warn) + timeout=timeout) if avail_url: LOG.debug("Using metadata source: '%s'", url2base[avail_url]) else: -- cgit v1.2.3 From 046352fc69e94f4c2f2d209ff5638fc455ff4b97 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Thu, 13 Feb 2014 21:24:41 +0000 Subject: GCE: add unit tests, user-data support and few other fixes --- cloudinit/sources/DataSourceGCE.py | 49 ++++++------ tests/unittests/test_datasource/test_gce.py | 112 ++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 24 deletions(-) create mode 100644 tests/unittests/test_datasource/test_gce.py (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 95a410ba..2d5ccad5 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -23,7 +23,7 @@ from cloudinit import url_helper LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' + 'metadata_url': 'http://169.254.169.254/computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') @@ -31,7 +31,7 @@ REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') class DataSourceGCE(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.metadata = {} + self.metadata = dict() self.ds_cfg = util.mergemanydict([ util.get_cfg_by_path(sys_cfg, ["datasource", "GCE"], {}), BUILTIN_DS_CONFIG]) @@ -59,33 +59,31 @@ class DataSourceGCE(sources.DataSource): 'user-data': self.metadata_address + 'instance/attributes/user-data', } - # if we cannot resolve the metadata server, then no point in trying - if not util.is_resolvable(self.metadata_address): - LOG.debug("%s is not resolvable", self.metadata_address) + # try to connect to metadata service or fail otherwise + try: + url_helper.readurl(url=url_map['instance-id'], headers=headers) + except url_helper.UrlError as e: + LOG.debug('Failed to connect to metadata service. Reason: {0}'.format( + e)) return False + # iterate over url_map keys to get metadata items for mkey in url_map.iterkeys(): try: - resp = url_helper.readurl(url=url_map[mkey], headers=headers, - retries=0) - except IOError: - return False - if resp.ok(): - if mkey == 'public-keys': - pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] - self.metadata[mkey] = pub_keys + resp = url_helper.readurl( + url=url_map[mkey], headers=headers) + if resp.code == 200: + if mkey == 'public-keys': + pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] + self.metadata[mkey] = pub_keys + else: + self.metadata[mkey] = resp.contents else: - self.metadata[mkey] = resp.contents - else: - if mkey in REQUIRED_FIELDS: - LOG.warn("required metadata '%s' not found in metadata", - url_map[mkey]) - return False - + self.metadata[mkey] = None + except url_helper.UrlError as e: self.metadata[mkey] = None - return False - - self.user_data_raw = self.metadata['user-data'] + LOG.warn('Failed to get {0} metadata item. Reason {1}.'.format( + mkey, e)) return True @property @@ -102,9 +100,12 @@ class DataSourceGCE(sources.DataSource): def get_hostname(self, fqdn=False): return self.metadata['local-hostname'] + def get_userdata_raw(self): + return self.metadata['user-data'] + @property def availability_zone(self): - return self.metadata['instance-zone'] + return self.metadata['availability-zone'] # Used to match classes to dependencies datasources = [ diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py new file mode 100644 index 00000000..0c58be52 --- /dev/null +++ b/tests/unittests/test_datasource/test_gce.py @@ -0,0 +1,112 @@ +# +# Copyright (C) 2014 Vaidas Jablonskis +# +# Author: Vaidas Jablonskis +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import unittest +import httpretty +import re + +from urlparse import urlparse + +from cloudinit import settings +from cloudinit import helpers +from cloudinit.sources import DataSourceGCE + +GCE_META = { + 'instance/id': '123', + 'instance/zone': 'foo/bar', + 'project/attributes/sshKeys': 'user:ssh-rsa AA2..+aRD0fyVw== root@server', + 'instance/hostname': 'server.project-name.local', + 'instance/attributes/user-data': '/bin/echo foo\n', +} + +GCE_META_PARTIAL = { + 'instance/id': '123', + 'instance/hostname': 'server.project-name.local', +} + +HEADERS = {'X-Google-Metadata-Request': 'True'} +MD_URL_RE = re.compile(r'http://169.254.169.254/computeMetadata/v1/.*') + + +def _request_callback(method, uri, headers): + url_path = urlparse(uri).path + if url_path.startswith('/computeMetadata/v1/'): + path = url_path.split('/computeMetadata/v1/')[1:][0] + else: + path = None + if path in GCE_META: + #return (200, headers, GCE_META.get(path)) + return (200, headers, GCE_META.get(path)) + else: + return (404, headers, '') + + +class TestDataSourceGCE(unittest.TestCase): + + def setUp(self): + self.ds = DataSourceGCE.DataSourceGCE( + settings.CFG_BUILTIN, None, + helpers.Paths({})) + + @httpretty.activate + def test_connection(self): + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + + success = self.ds.get_data() + self.assertTrue(success) + + req_header = httpretty.last_request().headers + self.assertDictContainsSubset(HEADERS, req_header) + + @httpretty.activate + def test_metadata(self): + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + self.ds.get_data() + + self.assertEqual(GCE_META.get('instance/hostname'), + self.ds.get_hostname()) + + self.assertEqual(GCE_META.get('instance/id'), + self.ds.get_instance_id()) + + self.assertEqual(GCE_META.get('instance/zone'), + self.ds.availability_zone) + + self.assertEqual(GCE_META.get('instance/attributes/user-data'), + self.ds.get_userdata_raw()) + + # we expect a list of public ssh keys with user names stripped + self.assertEqual(['ssh-rsa AA2..+aRD0fyVw== root@server'], + self.ds.get_public_ssh_keys()) + + # test partial metadata (missing user-data in particular) + @httpretty.activate + def test_metadata_partial(self): + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + self.ds.get_data() + + self.assertEqual(GCE_META_PARTIAL.get('instance/id'), + self.ds.get_instance_id()) + + self.assertEqual(GCE_META_PARTIAL.get('instance/hostname'), + self.ds.get_hostname()) -- cgit v1.2.3 From f40bdb4869567124dbc48feaa0574cc83df26e3a Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Thu, 13 Feb 2014 22:03:12 +0000 Subject: GCE: use dns name instead of IP address --- cloudinit/sources/DataSourceGCE.py | 11 ++++------- tests/unittests/test_datasource/test_gce.py | 3 +-- 2 files changed, 5 insertions(+), 9 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 2d5ccad5..e2be15b0 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -23,7 +23,7 @@ from cloudinit import url_helper LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://169.254.169.254/computeMetadata/v1/' + 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') @@ -59,12 +59,9 @@ class DataSourceGCE(sources.DataSource): 'user-data': self.metadata_address + 'instance/attributes/user-data', } - # try to connect to metadata service or fail otherwise - try: - url_helper.readurl(url=url_map['instance-id'], headers=headers) - except url_helper.UrlError as e: - LOG.debug('Failed to connect to metadata service. Reason: {0}'.format( - e)) + # if we cannot resolve the metadata server, then no point in trying + if not util.is_resolvable(self.metadata_address): + LOG.debug('{0} is not resolvable'.format(self.metadata_address)) return False # iterate over url_map keys to get metadata items diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 0c58be52..d91bd531 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -39,7 +39,7 @@ GCE_META_PARTIAL = { } HEADERS = {'X-Google-Metadata-Request': 'True'} -MD_URL_RE = re.compile(r'http://169.254.169.254/computeMetadata/v1/.*') +MD_URL_RE = re.compile(r'http://metadata.google.internal./computeMetadata/v1/.*') def _request_callback(method, uri, headers): @@ -49,7 +49,6 @@ def _request_callback(method, uri, headers): else: path = None if path in GCE_META: - #return (200, headers, GCE_META.get(path)) return (200, headers, GCE_META.get(path)) else: return (404, headers, '') -- cgit v1.2.3 From 6baa9166aa8b150566003d2d4cf56cfd03fe6952 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 21:09:47 -0500 Subject: some style changes, some pylint, be less noisy this changes url_map to a list and adds 'required' information. * If we've not already found an entry, and this is required, then debug log (ie, this is just not GCE). * if we already found an entry and this is required: warn split the keys fixing out of the loop. --- cloudinit/sources/DataSourceGCE.py | 65 ++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 21 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index e2be15b0..c993293f 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -51,37 +51,60 @@ class DataSourceGCE(sources.DataSource): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} - url_map = { - 'instance-id': self.metadata_address + 'instance/id', - 'availability-zone': self.metadata_address + 'instance/zone', - 'public-keys': self.metadata_address + 'project/attributes/sshKeys', - 'local-hostname': self.metadata_address + 'instance/hostname', - 'user-data': self.metadata_address + 'instance/attributes/user-data', - } + # url_map: (our-key, path, required) + url_map = [ + ('instance-id', 'instance/id', True), + ('availability-zone', 'instance/zone', True), + ('local-hostname', 'instance/hostname', True), + ('public-keys', 'project/attributes/sshKeys', False), + ('user-data', 'instance/attributes/user-data', False), + ] # if we cannot resolve the metadata server, then no point in trying if not util.is_resolvable(self.metadata_address): - LOG.debug('{0} is not resolvable'.format(self.metadata_address)) + LOG.debug("%s is not resolvable", self.metadata_address) return False # iterate over url_map keys to get metadata items - for mkey in url_map.iterkeys(): + found = False + for (mkey, path, required) in url_map: try: - resp = url_helper.readurl( - url=url_map[mkey], headers=headers) + resp = url_helper.readurl(url=self.metadata_address + path, + headers=headers) if resp.code == 200: - if mkey == 'public-keys': - pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] - self.metadata[mkey] = pub_keys - else: - self.metadata[mkey] = resp.contents + found = True + self.metadata[mkey] = resp.contents else: - self.metadata[mkey] = None + if required: + msg = "required url %s returned code %s. not GCE" + if not found: + LOG.debug(msg, path, resp.code) + else: + LOG.warn(msg, path, resp.code) + return False + else: + self.metadata[mkey] = None except url_helper.UrlError as e: + if required: + msg = "required url %s raised exception %s. not GCE" + if not found: + LOG.debug(msg, path, e) + else: + LOG.warn(msg, path, e) + return False + msg = "Failed to get %s metadata item: %s." + if found: + LOG.warn(msg, path, e) + else: + LOG.debug(msg, path, e) + self.metadata[mkey] = None - LOG.warn('Failed to get {0} metadata item. Reason {1}.'.format( - mkey, e)) - return True + + if self.metadata['public-keys']: + lines = self.metadata['public-keys'].splitlines() + self.metadata['public-keys'] = [self._trim_key(k) for k in lines] + + return found @property def launch_index(self): @@ -94,7 +117,7 @@ class DataSourceGCE(sources.DataSource): def get_public_ssh_keys(self): return self.metadata['public-keys'] - def get_hostname(self, fqdn=False): + def get_hostname(self, fqdn=False, _resolve_ip=False): return self.metadata['local-hostname'] def get_userdata_raw(self): -- cgit v1.2.3