diff options
Diffstat (limited to 'cloudinit/sources')
| -rw-r--r-- | cloudinit/sources/DataSourceConfigDrive.py | 28 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceDigitalOcean.py | 104 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceOVF.py | 3 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceOpenStack.py | 27 | ||||
| -rw-r--r-- | cloudinit/sources/helpers/openstack.py | 68 |
5 files changed, 190 insertions, 40 deletions
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 0c35f83a..27658073 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -37,7 +37,9 @@ DEFAULT_METADATA = { VALID_DSMODES = ("local", "net", "pass", "disabled") FS_TYPES = ('vfat', 'iso9660') LABEL_TYPES = ('config-2',) -OPTICAL_DEVICES = tuple(('/dev/sr%s' % i for i in range(0, 2))) +POSSIBLE_MOUNTS = ('sr', 'cd') +OPTICAL_DEVICES = tuple(('/dev/%s%s' % (z, i) for z in POSSIBLE_MOUNTS + for i in range(0, 2))) class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): @@ -70,7 +72,15 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): if not found: for dev in find_candidate_devs(): try: - results = util.mount_cb(dev, read_config_drive) + # Set mtype if freebsd and turn off sync + if dev.startswith("/dev/cd"): + mtype = "cd9660" + sync = False + else: + mtype = None + sync = True + results = util.mount_cb(dev, read_config_drive, mtype=mtype, + sync=sync) found = dev except openstack.NonReadable: pass @@ -125,7 +135,15 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): self.userdata_raw = results.get('userdata') self.version = results['version'] self.files.update(results.get('files', {})) - self.vendordata_raw = results.get('vendordata') + + vd = results.get('vendordata') + self.vendordata_pure = vd + try: + self.vendordata_raw = openstack.convert_vendordata_json(vd) + except ValueError as e: + LOG.warn("Invalid content in vendor-data: %s", e) + self.vendordata_raw = None + return True @@ -160,10 +178,10 @@ def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): return "net" -def read_config_drive(source_dir, version="2012-08-10"): +def read_config_drive(source_dir): reader = openstack.ConfigDriveReader(source_dir) finders = [ - (reader.read_v2, [], {'version': version}), + (reader.read_v2, [], {}), (reader.read_v1, [], {}), ] excps = [] diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py new file mode 100644 index 00000000..069bdb41 --- /dev/null +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -0,0 +1,104 @@ +# vi: ts=4 expandtab +# +# Author: Neal Shrader <neal@digitalocean.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from cloudinit import log as logging +from cloudinit import util +from cloudinit import sources +from cloudinit import ec2_utils +from types import StringType +import functools + + +LOG = logging.getLogger(__name__) + +BUILTIN_DS_CONFIG = { + 'metadata_url': 'http://169.254.169.254/metadata/v1/', + 'mirrors_url': 'http://mirrors.digitalocean.com/' +} +MD_RETRIES = 0 +MD_TIMEOUT = 1 + +class DataSourceDigitalOcean(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.metadata = dict() + self.ds_cfg = util.mergemanydict([ + util.get_cfg_by_path(sys_cfg, ["datasource", "DigitalOcean"], {}), + BUILTIN_DS_CONFIG]) + self.metadata_address = self.ds_cfg['metadata_url'] + + if self.ds_cfg.get('retries'): + self.retries = self.ds_cfg['retries'] + else: + self.retries = MD_RETRIES + + if self.ds_cfg.get('timeout'): + self.timeout = self.ds_cfg['timeout'] + else: + self.timeout = MD_TIMEOUT + + def get_data(self): + caller = functools.partial(util.read_file_or_url, timeout=self.timeout, + retries=self.retries) + md = ec2_utils.MetadataMaterializer(str(caller(self.metadata_address)), + base_url=self.metadata_address, + caller=caller) + + self.metadata = md.materialize() + + if self.metadata.get('id'): + return True + else: + return False + + def get_userdata_raw(self): + return "\n".join(self.metadata['user-data']) + + def get_vendordata_raw(self): + return "\n".join(self.metadata['vendor-data']) + + def get_public_ssh_keys(self): + if type(self.metadata['public-keys']) is StringType: + return [self.metadata['public-keys']] + else: + return self.metadata['public-keys'] + + @property + def availability_zone(self): + return self.metadata['region'] + + def get_instance_id(self): + return self.metadata['id'] + + def get_hostname(self, fqdn=False): + return self.metadata['hostname'] + + def get_package_mirror_info(self): + return self.ds_cfg['mirrors_url'] + + @property + def launch_index(self): + return None + +# Used to match classes to dependencies +datasources = [ + (DataSourceDigitalOcean, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + ] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 2f53c1ba..7ba60735 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -215,8 +215,7 @@ def transport_iso9660(require_iso=True): continue try: - (fname, contents) = util.mount_cb(fullp, - get_ovf_env, mtype=mtype) + (fname, contents) = util.mount_cb(fullp, get_ovf_env, mtype=mtype) except util.MountFailedError: LOG.debug("%s not mountable as iso9660" % fullp) continue diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 0970d07b..469c2e2a 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -88,11 +88,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): md_urls = [] url2base = {} for url in urls: - for version in openstack.OS_VERSIONS + (openstack.OS_LATEST,): - md_url = url_helper.combine_url(url, 'openstack', - version, 'meta_data.json') - md_urls.append(md_url) - url2base[md_url] = url + md_url = url_helper.combine_url(url, 'openstack') + md_urls.append(md_url) + url2base[md_url] = url (max_wait, timeout) = self._get_url_settings() start_time = time.time() @@ -119,8 +117,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'Crawl of openstack metadata service', read_metadata_service, args=[self.metadata_address], - kwargs={'ssl_details': self.ssl_details, - 'version': openstack.OS_HAVANA}) + kwargs={'ssl_details': self.ssl_details}) except openstack.NonReadable: return False except (openstack.BrokenMetadata, IOError): @@ -143,20 +140,20 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.version = results['version'] self.files.update(results.get('files', {})) - # if vendordata includes 'cloud-init', then read that explicitly - # for cloud-init (for namespacing). vd = results.get('vendordata') - if isinstance(vd, dict) and 'cloud-init' in vd: - self.vendordata_raw = vd['cloud-init'] - else: - self.vendordata_raw = vd + self.vendordata_pure = vd + try: + self.vendordata_raw = openstack.convert_vendordata_json(vd) + except ValueError as e: + LOG.warn("Invalid content in vendor-data: %s", e) + self.vendordata_raw = None return True -def read_metadata_service(base_url, version=None, ssl_details=None): +def read_metadata_service(base_url, ssl_details=None): reader = openstack.MetadataReader(base_url, ssl_details=ssl_details) - return reader.read_v2(version=version) + return reader.read_v2() # Used to match classes to dependencies diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 3c6bb6aa..b7e19314 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -21,6 +21,7 @@ import abc import base64 import copy +import functools import os from cloudinit import ec2_utils @@ -48,6 +49,7 @@ OS_LATEST = 'latest' OS_FOLSOM = '2012-08-10' OS_GRIZZLY = '2013-04-04' OS_HAVANA = '2013-10-17' +# keep this in chronological order. new supported versions go at the end. OS_VERSIONS = ( OS_FOLSOM, OS_GRIZZLY, @@ -161,25 +163,27 @@ class BaseReader(object): def _read_ec2_metadata(self): pass - def _find_working_version(self, version): + def _find_working_version(self): try: - versions_available = self._fetch_available_versions(self) + versions_available = self._fetch_available_versions() except Exception as e: - LOG.warn("Unable to read openstack versions from %s due to: %s", - self.base_path, e) + LOG.debug("Unable to read openstack versions from %s due to: %s", + self.base_path, e) versions_available = [] - search_versions = [version] + list(OS_VERSIONS) + # openstack.OS_VERSIONS is stored in chronological order, so + # reverse it to check newest first. + supported = [v for v in reversed(list(OS_VERSIONS))] selected_version = OS_LATEST - for potential_version in search_versions: + + for potential_version in supported: if potential_version not in versions_available: continue selected_version = potential_version break - if selected_version != version: - LOG.warn("Version '%s' not available, attempting to use" - " version '%s' instead", version, selected_version) + LOG.debug("Selected version '%s' from %s", selected_version, + versions_available) return selected_version def _read_content_path(self, item): @@ -191,7 +195,7 @@ class BaseReader(object): path = self._path_join(self.base_path, "openstack", *path_pieces) return self._path_read(path) - def read_v2(self, version=None): + def read_v2(self): """Reads a version 2 formatted location. Return a dict with metadata, userdata, ec2-metadata, dsmode, @@ -200,6 +204,9 @@ class BaseReader(object): If not a valid location, raise a NonReadable exception. """ + load_json_anytype = functools.partial( + util.load_json, root_types=(dict, basestring, list)) + def datafiles(version): files = {} files['metadata'] = ( @@ -218,16 +225,15 @@ class BaseReader(object): files['vendordata'] = ( self._path_join("openstack", version, 'vendor_data.json'), False, - util.load_json, + load_json_anytype, ) return files - version = self._find_working_version(version) results = { 'userdata': '', 'version': 2, } - data = datafiles(version) + data = datafiles(self._find_working_version()) for (name, (path, required, translator)) in data.iteritems(): path = self._path_join(self.base_path, path) data = None @@ -239,7 +245,8 @@ class BaseReader(object): LOG.debug("Failed reading optional path %s due" " to: %s", path, e) else: - LOG.exception("Failed reading mandatory path %s", path) + LOG.debug("Failed reading mandatory path %s due" + " to: %s", path, e) else: found = True if required and not found: @@ -325,7 +332,7 @@ class ConfigDriveReader(BaseReader): path = self._path_join(self.base_path, 'openstack') found = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path))] - self._versions = tuple(found) + self._versions = found return self._versions def _read_ec2_metadata(self): @@ -418,18 +425,18 @@ class MetadataReader(BaseReader): def _fetch_available_versions(self): # <baseurl>/openstack/ returns a newline separated list of versions if self._versions is not None: - return self.os_versions + return self._versions found = [] + version_path = self._path_join(self.base_path, "openstack") content = self._path_read(version_path) for line in content.splitlines(): line = line.strip() if not line: continue found.append(line) - self._versions = tuple(found) + self._versions = found return self._versions - def _path_read(self, path): def should_retry_cb(_request_args, cause): @@ -456,3 +463,28 @@ class MetadataReader(BaseReader): return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details, timeout=self.timeout, retries=self.retries) + + +def convert_vendordata_json(data, recurse=True): + """ data: a loaded json *object* (strings, arrays, dicts). + return something suitable for cloudinit vendordata_raw. + + if data is: + None: return None + string: return string + list: return data + the list is then processed in UserDataProcessor + dict: return convert_vendordata_json(data.get('cloud-init')) + """ + if not data: + return None + if isinstance(data, (str, unicode, basestring)): + return data + if isinstance(data, list): + return copy.deepcopy(data) + if isinstance(data, dict): + if recurse is True: + return convert_vendordata_json(data.get('cloud-init'), + recurse=False) + raise ValueError("vendordata['cloud-init'] cannot be dict") + raise ValueError("Unknown data type for vendordata: %s" % type(data)) |