From c104d6dfa464a8906c16b4f09b4b76ab5bf2e4e1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 1 Feb 2014 22:48:55 -0800 Subject: Add a openstack specific datasource Openstack has a unique derivative datasource that is gaining usage. Previously the config drive datasource provided part of this functionality as well as the ec2 datasource, but since new functionality is being added to openstack is seems benefical to combine the used parts into one datasource just made for handling openstack deployments. This patch factors out the common logic shared between the config drive and the openstack metadata datasource and places that in a shared helper file and then creates a new openstack datasource that readers from the openstack metadata service and refactors the config drive datasource to use this common logic. --- .../unittests/test_datasource/test_configdrive.py | 35 ++++++++++++---------- tests/unittests/test_ec2_util.py | 35 +++++++++++----------- 2 files changed, 37 insertions(+), 33 deletions(-) (limited to 'tests') diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index d5935294..bd6cdd5d 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -9,6 +9,7 @@ from mocker import MockerTestCase from cloudinit import helpers from cloudinit import settings from cloudinit.sources import DataSourceConfigDrive as ds +from cloudinit.sources.helpers import openstack from cloudinit import util from tests.unittests import helpers as unit_helpers @@ -71,7 +72,7 @@ class TestConfigDriveDataSource(MockerTestCase): def test_ec2_metadata(self): populate_dir(self.tmp, CFG_DRIVE_FILES_V2) - found = ds.read_config_drive_dir(self.tmp) + found = ds.read_config_drive(self.tmp) self.assertTrue('ec2-metadata' in found) ec2_md = found['ec2-metadata'] self.assertEqual(EC2_META, ec2_md) @@ -81,7 +82,7 @@ class TestConfigDriveDataSource(MockerTestCase): cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, helpers.Paths({})) - found = ds.read_config_drive_dir(self.tmp) + found = ds.read_config_drive(self.tmp) cfg_ds.metadata = found['metadata'] name_tests = { 'ami': '/dev/vda1', @@ -112,7 +113,7 @@ class TestConfigDriveDataSource(MockerTestCase): cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, helpers.Paths({})) - found = ds.read_config_drive_dir(self.tmp) + found = ds.read_config_drive(self.tmp) os_md = found['metadata'] cfg_ds.metadata = os_md name_tests = { @@ -140,7 +141,7 @@ class TestConfigDriveDataSource(MockerTestCase): cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, helpers.Paths({})) - found = ds.read_config_drive_dir(self.tmp) + found = ds.read_config_drive(self.tmp) ec2_md = found['ec2-metadata'] os_md = found['metadata'] cfg_ds.ec2_metadata = ec2_md @@ -165,13 +166,13 @@ class TestConfigDriveDataSource(MockerTestCase): my_mock.replay() device = cfg_ds.device_name_to_device(name) self.assertEquals(dev_name, device) - + def test_dev_ec2_map(self): populate_dir(self.tmp, CFG_DRIVE_FILES_V2) cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, helpers.Paths({})) - found = ds.read_config_drive_dir(self.tmp) + found = ds.read_config_drive(self.tmp) exists_mock = self.mocker.replace(os.path.exists, spec=False, passthrough=False) exists_mock(mocker.ARGS) @@ -200,10 +201,11 @@ class TestConfigDriveDataSource(MockerTestCase): populate_dir(self.tmp, CFG_DRIVE_FILES_V2) - found = ds.read_config_drive_dir(self.tmp) + found = ds.read_config_drive(self.tmp) expected_md = copy(OSTACK_META) expected_md['instance-id'] = expected_md['uuid'] + expected_md['local-hostname'] = expected_md['hostname'] self.assertEqual(USER_DATA, found['userdata']) self.assertEqual(expected_md, found['metadata']) @@ -219,10 +221,11 @@ class TestConfigDriveDataSource(MockerTestCase): populate_dir(self.tmp, data) - found = ds.read_config_drive_dir(self.tmp) + found = ds.read_config_drive(self.tmp) expected_md = copy(OSTACK_META) expected_md['instance-id'] = expected_md['uuid'] + expected_md['local-hostname'] = expected_md['hostname'] self.assertEqual(expected_md, found['metadata']) @@ -235,8 +238,8 @@ class TestConfigDriveDataSource(MockerTestCase): populate_dir(self.tmp, data) - self.assertRaises(ds.BrokenConfigDriveDir, - ds.read_config_drive_dir, self.tmp) + self.assertRaises(openstack.BrokenMetadata, + ds.read_config_drive, self.tmp) def test_seed_dir_no_configdrive(self): """Verify that no metadata raises NonConfigDriveDir.""" @@ -247,14 +250,14 @@ class TestConfigDriveDataSource(MockerTestCase): data["openstack/latest/random-file.txt"] = "random-content" data["content/foo"] = "foocontent" - self.assertRaises(ds.NonConfigDriveDir, - ds.read_config_drive_dir, my_d) + self.assertRaises(openstack.NonReadable, + ds.read_config_drive, my_d) def test_seed_dir_missing(self): """Verify that missing seed_dir raises NonConfigDriveDir.""" my_d = os.path.join(self.tmp, "nonexistantdirectory") - self.assertRaises(ds.NonConfigDriveDir, - ds.read_config_drive_dir, my_d) + self.assertRaises(openstack.NonReadable, + ds.read_config_drive, my_d) def test_find_candidates(self): devs_with_answers = {} @@ -303,7 +306,7 @@ class TestConfigDriveDataSource(MockerTestCase): def cfg_ds_from_dir(seed_d): - found = ds.read_config_drive_dir(seed_d) + found = ds.read_config_drive(seed_d) cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, helpers.Paths({})) populate_ds_from_read_config(cfg_ds, seed_d, found) @@ -318,7 +321,7 @@ def populate_ds_from_read_config(cfg_ds, source, results): cfg_ds.metadata = results.get('metadata') cfg_ds.ec2_metadata = results.get('ec2-metadata') cfg_ds.userdata_raw = results.get('userdata') - cfg_ds.version = results.get('cfgdrive_ver') + cfg_ds.version = results.get('version') def populate_dir(seed_dir, files): diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index dd588aca..18d36d86 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -1,6 +1,7 @@ from tests.unittests import helpers from cloudinit import ec2_utils as eu +from cloudinit import url_helper as uh import httpretty as hp @@ -40,11 +41,11 @@ class TestEc2Util(helpers.TestCase): body="\n".join(['hostname', 'instance-id', 'ami-launch-index'])) - hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'), status=200, body='ec2.fake.host.name.com') - hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'), status=200, body='123') - hp.register_uri(hp.GET, eu.combine_url(base_url, 'ami-launch-index'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'ami-launch-index'), status=200, body='1') md = eu.get_instance_metadata(self.VERSION, retries=0) self.assertEquals(md['hostname'], 'ec2.fake.host.name.com') @@ -58,14 +59,14 @@ class TestEc2Util(helpers.TestCase): body="\n".join(['hostname', 'instance-id', 'public-keys/'])) - hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'), status=200, body='ec2.fake.host.name.com') - hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'), status=200, body='123') - hp.register_uri(hp.GET, eu.combine_url(base_url, 'public-keys/'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'public-keys/'), status=200, body='0=my-public-key') hp.register_uri(hp.GET, - eu.combine_url(base_url, 'public-keys/0/openssh-key'), + uh.combine_url(base_url, 'public-keys/0/openssh-key'), status=200, body='ssh-rsa AAAA.....wZEf my-public-key') md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) self.assertEquals(md['hostname'], 'ec2.fake.host.name.com') @@ -79,18 +80,18 @@ class TestEc2Util(helpers.TestCase): body="\n".join(['hostname', 'instance-id', 'public-keys/'])) - hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'), status=200, body='ec2.fake.host.name.com') - hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'), status=200, body='123') - hp.register_uri(hp.GET, eu.combine_url(base_url, 'public-keys/'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'public-keys/'), status=200, body="\n".join(['0=my-public-key', '1=my-other-key'])) hp.register_uri(hp.GET, - eu.combine_url(base_url, 'public-keys/0/openssh-key'), + uh.combine_url(base_url, 'public-keys/0/openssh-key'), status=200, body='ssh-rsa AAAA.....wZEf my-public-key') hp.register_uri(hp.GET, - eu.combine_url(base_url, 'public-keys/1/openssh-key'), + uh.combine_url(base_url, 'public-keys/1/openssh-key'), status=200, body='ssh-rsa AAAA.....wZEf my-other-key') md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) self.assertEquals(md['hostname'], 'ec2.fake.host.name.com') @@ -104,20 +105,20 @@ class TestEc2Util(helpers.TestCase): body="\n".join(['hostname', 'instance-id', 'block-device-mapping/'])) - hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'), status=200, body='ec2.fake.host.name.com') - hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'), status=200, body='123') hp.register_uri(hp.GET, - eu.combine_url(base_url, 'block-device-mapping/'), + uh.combine_url(base_url, 'block-device-mapping/'), status=200, body="\n".join(['ami', 'ephemeral0'])) hp.register_uri(hp.GET, - eu.combine_url(base_url, 'block-device-mapping/ami'), + uh.combine_url(base_url, 'block-device-mapping/ami'), status=200, body="sdb") hp.register_uri(hp.GET, - eu.combine_url(base_url, + uh.combine_url(base_url, 'block-device-mapping/ephemeral0'), status=200, body="sdc") -- cgit v1.2.3 From 810df2c55c108e7e4064263e508d9786d8b1dc8e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 1 Feb 2014 22:58:58 -0800 Subject: Don't forget the rest of the files! --- cloudinit/sources/DataSourceOpenStack.py | 163 +++++++++ cloudinit/sources/helpers/__init__.py | 14 + cloudinit/sources/helpers/openstack.py | 420 ++++++++++++++++++++++ tests/unittests/test_datasource/test_openstack.py | 122 +++++++ 4 files changed, 719 insertions(+) create mode 100644 cloudinit/sources/DataSourceOpenStack.py create mode 100644 cloudinit/sources/helpers/__init__.py create mode 100644 cloudinit/sources/helpers/openstack.py create mode 100644 tests/unittests/test_datasource/test_openstack.py (limited to 'tests') diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py new file mode 100644 index 00000000..44889f4e --- /dev/null +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -0,0 +1,163 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import time + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import url_helper +from cloudinit import util + +from cloudinit.sources.helpers import openstack + +LOG = logging.getLogger(__name__) + +# Various defaults/constants... +DEF_MD_URL = "http://169.254.169.254" +DEFAULT_IID = "iid-dsopenstack" +DEFAULT_METADATA = { + "instance-id": DEFAULT_IID, +} +VALID_DSMODES = ("net", "disabled") + + +class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths) + self.dsmode = 'net' + self.metadata_address = None + self.ssl_details = util.fetch_ssl_details(self.paths) + self.version = None + self.files = {} + + def __str__(self): + root = sources.DataSource.__str__(self) + mstr = "%s [%s,ver=%s]" % (root, self.dsmode, self.version) + return mstr + + def _get_url_settings(self): + # TODO(harlowja): this is shared with ec2 datasource, we should just + # move it to a shared location instead... + ds_cfg = self.ds_cfg + if not ds_cfg: + ds_cfg = {} + max_wait = 120 + try: + max_wait = int(ds_cfg.get("max_wait", max_wait)) + except Exception: + util.logexc(LOG, "Failed to get max wait. using %s", max_wait) + + timeout = 50 + try: + timeout = max(0, int(ds_cfg.get("timeout", timeout))) + except Exception: + util.logexc(LOG, "Failed to get timeout, using %s", timeout) + return (max_wait, timeout) + + def wait_for_metadata_service(self): + ds_cfg = self.ds_cfg + if not ds_cfg: + ds_cfg = {} + urls = ds_cfg.get("metadata_urls", [DEF_MD_URL]) + filtered = [x for x in urls if util.is_resolvable_url(x)] + if set(filtered) != set(urls): + LOG.debug("Removed the following from metadata urls: %s", + list((set(urls) - set(filtered)))) + if len(filtered): + urls = filtered + else: + LOG.warn("Empty metadata url list! using default list") + urls = [DEF_MD_URL] + + md_urls = [] + url2base = {} + for url in urls: + md_url = url_helper.combine_url(url, 'openstack', + openstack.OS_LATEST, + 'meta_data.json') + md_urls.append(md_url) + url2base[md_url] = url + + (max_wait, timeout) = self._get_url_settings() + if max_wait <= 0: + return False + start_time = time.time() + avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait, + timeout=timeout, + status_cb=LOG.warn) + if avail_url: + LOG.debug("Using metadata source: '%s'", url2base[avail_url]) + else: + LOG.critical("Giving up on md from %s after %s seconds", + md_urls, int(time.time() - start_time)) + + self.metadata_address = url2base.get(avail_url) + return bool(avail_url) + + def get_data(self): + try: + if not self.wait_for_metadata_service(): + return False + except IOError: + return False + + try: + results = util.log_time(LOG.debug, + 'Crawl of openstack metadata service', + read_metadata_service, + args=[self.metadata_address], + kwargs={'ssl_details': self.ssl_details}) + except openstack.NonReadable: + return False + except openstack.BrokenMetadata: + util.logexc(LOG, "Broken metadata address %s", + self.metadata_address) + return False + + user_dsmode = results.get('dsmode', None) + if user_dsmode not in VALID_DSMODES + (None,): + LOG.warn("User specified invalid mode: %s" % user_dsmode) + user_dsmode = None + if user_dsmode == 'disabled': + return False + + md = results.get('metadata', {}) + md = util.mergemanydict([md, DEFAULT_METADATA]) + self.metadata = md + self.ec2_metadata = results.get('ec2-metadata') + self.userdata_raw = results.get('userdata') + self.version = results['version'] + self.files.update(results.get('files', {})) + self.vendordata_raw = results.get('vendordata') + return True + + +def read_metadata_service(base_url, version=None, ssl_details=None): + reader = openstack.MetadataReader(base_url, ssl_details=ssl_details) + return reader.read_v2(version=version) + + +# Used to match classes to dependencies +datasources = [ + (DataSourceOpenStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/helpers/__init__.py b/cloudinit/sources/helpers/__init__.py new file mode 100644 index 00000000..2cf99ec8 --- /dev/null +++ b/cloudinit/sources/helpers/__init__.py @@ -0,0 +1,14 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py new file mode 100644 index 00000000..9dbef677 --- /dev/null +++ b/cloudinit/sources/helpers/openstack.py @@ -0,0 +1,420 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import abc +import base64 +import copy +import functools +import os + +from cloudinit import ec2_utils +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import url_helper +from cloudinit import util + +# For reference: http://tinyurl.com/laora4c + +LOG = logging.getLogger(__name__) + +FILES_V1 = { + # Path <-> (metadata key name, translator function, default value) + 'etc/network/interfaces': ('network_config', lambda x: x, ''), + 'meta.js': ('meta_js', util.load_json, {}), + "root/.ssh/authorized_keys": ('authorized_keys', lambda x: x, ''), +} +KEY_COPIES = ( + # Cloud-init metadata names <-> (metadata key, is required) + ('local-hostname', 'hostname', False), + ('instance-id', 'uuid', True), +) +OS_VERSIONS = ( + '2012-08-10', # folsom + '2013-04-04', # grizzly + '2013-10-17', # havana +) +OS_LATEST = 'latest' + + +class NonReadable(IOError): + pass + + +class BrokenMetadata(IOError): + pass + + +class SourceMixin(object): + def _ec2_name_to_device(self, name): + if not self.ec2_metadata: + return None + bdm = self.ec2_metadata.get('block-device-mapping', {}) + for (ent_name, device) in bdm.items(): + if name == ent_name: + return device + return None + + def get_public_ssh_keys(self): + name = "public_keys" + if self.version == 1: + name = "public-keys" + return sources.normalize_pubkey_data(self.metadata.get(name)) + + def _os_name_to_device(self, name): + device = None + try: + criteria = 'LABEL=%s' % (name) + if name == 'swap': + criteria = 'TYPE=%s' % (name) + dev_entries = util.find_devs_with(criteria) + if dev_entries: + device = dev_entries[0] + except util.ProcessExecutionError: + pass + return device + + def _validate_device_name(self, device): + if not device: + return None + if not device.startswith("/"): + device = "/dev/%s" % device + if os.path.exists(device): + return device + # Durn, try adjusting the mapping + remapped = self._remap_device(os.path.basename(device)) + if remapped: + LOG.debug("Remapped device name %s => %s", device, remapped) + return remapped + return None + + def device_name_to_device(self, name): + # Translate a 'name' to a 'physical' device + if not name: + return None + # Try the ec2 mapping first + names = [name] + if name == 'root': + names.insert(0, 'ami') + if name == 'ami': + names.append('root') + device = None + LOG.debug("Using ec2 style lookup to find device %s", names) + for n in names: + device = self._ec2_name_to_device(n) + device = self._validate_device_name(device) + if device: + break + # Try the openstack way second + if not device: + LOG.debug("Using openstack style lookup to find device %s", names) + for n in names: + device = self._os_name_to_device(n) + device = self._validate_device_name(device) + if device: + break + # Ok give up... + if not device: + return None + else: + LOG.debug("Mapped %s to device %s", name, device) + return device + + +class BaseReader(object): + __metaclass__ = abc.ABCMeta + + def __init__(self, base_path): + self.base_path = base_path + + @abc.abstractmethod + def _path_join(self, base, *add_ons): + pass + + @abc.abstractmethod + def _path_exists(self, path): + pass + + @abc.abstractmethod + def _path_read(self, path): + pass + + @abc.abstractmethod + def _read_ec2_metadata(self): + pass + + def _read_content_path(self, item): + path = item.get('content_path', '').lstrip("/") + path_pieces = path.split("/") + valid_pieces = [p for p in path_pieces if len(p)] + if not valid_pieces: + raise BrokenMetadata("Item %s has no valid content path" % (item)) + path = self._path_join(self.base_path, "openstack", *path_pieces) + return self._path_read(path) + + def _find_working_version(self, version): + search_versions = [version] + list(OS_VERSIONS) + for potential_version in search_versions: + if not potential_version: + continue + path = self._path_join(self.base_path, "openstack", + potential_version) + if self._path_exists(path): + if potential_version != version: + LOG.warn("Version '%s' not available, attempting to use" + " version '%s' instead", version, + potential_version) + return potential_version + LOG.warn("Version '%s' not available, attempting to use '%s'" + " instead", version, OS_LATEST) + return OS_LATEST + + def read_v2(self, version=None): + """Reads a version 2 formatted location. + + Return a dict with metadata, userdata, ec2-metadata, dsmode, + network_config, files and version (2). + + If not a valid location, raise a NonReadable exception. + """ + + def datafiles(version): + files = {} + files['metadata'] = ( + # File path to read + self._path_join("openstack", version, 'meta_data.json'), + # Is it required? + True, + # Translator function (applied after loading) + util.load_json, + ) + files['userdata'] = ( + self._path_join("openstack", version, 'user_data'), + False, + lambda x: x, + ) + files['vendordata'] = ( + self._path_join("openstack", version, 'vendor_data.json'), + False, + util.load_json, + ) + return files + + version = self._find_working_version(version) + results = { + 'userdata': '', + 'version': 2, + } + data = datafiles(version) + for (name, (path, required, translator)) in data.iteritems(): + path = self._path_join(self.base_path, path) + data = None + found = False + if self._path_exists(path): + try: + data = self._path_read(path) + except IOError: + raise NonReadable("Failed to read: %s" % path) + found = True + else: + if required: + raise NonReadable("Missing mandatory path: %s" % path) + if found and translator: + try: + data = translator(data) + except Exception as e: + raise BrokenMetadata("Failed to process " + "path %s: %s" % (path, e)) + if found: + results[name] = data + + metadata = results['metadata'] + if 'random_seed' in metadata: + random_seed = metadata['random_seed'] + try: + metadata['random_seed'] = base64.b64decode(random_seed) + except (ValueError, TypeError) as e: + raise BrokenMetadata("Badly formatted metadata" + " random_seed entry: %s" % e) + + # load any files that were provided + files = {} + metadata_files = metadata.get('files', []) + for item in metadata_files: + if 'path' not in item: + continue + path = item['path'] + try: + files[path] = self._read_content_path(item) + except Exception as e: + raise BrokenMetadata("Failed to read provided " + "file %s: %s" % (path, e)) + results['files'] = files + + # The 'network_config' item in metadata is a content pointer + # to the network config that should be applied. It is just a + # ubuntu/debian '/etc/network/interfaces' file. + net_item = metadata.get("network_config", None) + if net_item: + try: + results['network_config'] = self._read_content_path(net_item) + except IOError as e: + raise BrokenMetadata("Failed to read network" + " configuration: %s" % (e)) + + # To openstack, user can specify meta ('nova boot --meta=key=value') + # and those will appear under metadata['meta']. + # if they specify 'dsmode' they're indicating the mode that they intend + # for this datasource to operate in. + try: + results['dsmode'] = metadata['meta']['dsmode'] + except KeyError: + pass + + # Read any ec2-metadata (if applicable) + results['ec2-metadata'] = self._read_ec2_metadata() + + # Perform some misc. metadata key renames... + for (target_key, source_key, is_required) in KEY_COPIES: + if is_required and source_key not in metadata: + raise BrokenMetadata("No '%s' entry in metadata" % source_key) + if source_key in metadata: + metadata[target_key] = metadata.get(source_key) + return results + + +class ConfigDriveReader(BaseReader): + def __init__(self, base_path): + super(ConfigDriveReader, self).__init__(base_path) + + def _path_join(self, base, *add_ons): + components = [base] + list(add_ons) + return os.path.join(*components) + + def _path_exists(self, path): + return os.path.exists(path) + + def _path_read(self, path): + return util.load_file(path) + + def _read_ec2_metadata(self): + path = self._path_join(self.base_path, + 'ec2', 'latest', 'meta-data.json') + if not self._path_exists(path): + return {} + else: + try: + return util.load_json(self._path_read(path)) + except Exception as e: + raise BrokenMetadata("Failed to process " + "path %s: %s" % (path, e)) + + def read_v1(self): + """Reads a version 1 formatted location. + + Return a dict with metadata, userdata, dsmode, files and version (1). + + If not a valid path, raise a NonReadable exception. + """ + + found = {} + for name in FILES_V1.keys(): + path = self._path_join(self.base_path, name) + if self._path_exists(path): + found[name] = path + if len(found) == 0: + raise NonReadable("%s: no files found" % (self.base_path)) + + md = {} + for (name, (key, translator, default)) in FILES_V1.iteritems(): + if name in found: + path = found[name] + try: + contents = self._path_read(path) + except IOError: + raise BrokenMetadata("Failed to read: %s" % path) + try: + md[key] = translator(contents) + except Exception as e: + raise BrokenMetadata("Failed to process " + "path %s: %s" % (path, e)) + else: + md[key] = copy.deepcopy(default) + + keydata = md['authorized_keys'] + meta_js = md['meta_js'] + + # keydata in meta_js is preferred over "injected" + keydata = meta_js.get('public-keys', keydata) + if keydata: + lines = keydata.splitlines() + md['public-keys'] = [l for l in lines + if len(l) and not l.startswith("#")] + + # config-drive-v1 has no way for openstack to provide the instance-id + # so we copy that into metadata from the user input + if 'instance-id' in meta_js: + md['instance-id'] = meta_js['instance-id'] + + results = { + 'version': 1, + 'metadata': md, + } + + # allow the user to specify 'dsmode' in a meta tag + if 'dsmode' in meta_js: + results['dsmode'] = meta_js['dsmode'] + + # config-drive-v1 has no way of specifying user-data, so the user has + # to cheat and stuff it in a meta tag also. + results['userdata'] = meta_js.get('user-data', '') + + # this implementation does not support files other than + # network/interfaces and authorized_keys... + results['files'] = {} + + return results + + +class MetadataReader(BaseReader): + def __init__(self, base_url, ssl_details=None, timeout=5, retries=5): + super(MetadataReader, self).__init__(base_url) + self._url_reader = functools.partial(url_helper.readurl, + retries=retries, + ssl_details=ssl_details, + timeout=timeout) + self._url_checker = functools.partial(url_helper.existsurl, + ssl_details=ssl_details, + timeout=timeout) + self._ec2_reader = functools.partial(ec2_utils.get_instance_metadata, + ssl_details=ssl_details, + timeout=timeout, + retries=retries) + + def _path_read(self, path): + return str(self._url_reader(path)) + + def _path_exists(self, path): + return self._url_checker(path) + + def _path_join(self, base, *add_ons): + return url_helper.combine_url(base, *add_ons) + + def _read_ec2_metadata(self): + return self._ec2_reader() diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py new file mode 100644 index 00000000..7d93f1d3 --- /dev/null +++ b/tests/unittests/test_datasource/test_openstack.py @@ -0,0 +1,122 @@ +import re +import json + +from StringIO import StringIO + +from urlparse import urlparse + +from tests.unittests import helpers + +from cloudinit.sources import DataSourceOpenStack as ds +from cloudinit.sources.helpers import openstack +from cloudinit import util + +import httpretty as hp + +BASE_URL = "http://169.254.169.254" +PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' +EC2_META = { + 'ami-id': 'ami-00000001', + 'ami-launch-index': 0, + 'ami-manifest-path': 'FIXME', + 'hostname': 'sm-foo-test.novalocal', + 'instance-action': 'none', + 'instance-id': 'i-00000001', + 'instance-type': 'm1.tiny', + 'local-hostname': 'sm-foo-test.novalocal', + 'local-ipv4': '0.0.0.0', + 'public-hostname': 'sm-foo-test.novalocal', + 'public-ipv4': '0.0.0.1', + 'reservation-id': 'r-iru5qm4m', +} +USER_DATA = '#!/bin/sh\necho This is user data\n' +VENDOR_DATA = { + 'magic': '', +} +OSTACK_META = { + 'availability_zone': 'nova', + 'files': [{'content_path': '/content/0000', 'path': '/etc/foo.cfg'}, + {'content_path': '/content/0001', 'path': '/etc/bar/bar.cfg'}], + 'hostname': 'sm-foo-test.novalocal', + 'meta': {'dsmode': 'local', 'my-meta': 'my-value'}, + 'name': 'sm-foo-test', + 'public_keys': {'mykey': PUBKEY}, + 'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c', +} +CONTENT_0 = 'This is contents of /etc/foo.cfg\n' +CONTENT_1 = '# this is /etc/bar/bar.cfg\n' +OS_FILES = { + 'openstack/2012-08-10/meta_data.json': json.dumps(OSTACK_META), + 'openstack/2012-08-10/user_data': USER_DATA, + 'openstack/content/0000': CONTENT_0, + 'openstack/content/0001': CONTENT_1, + 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), + 'openstack/latest/user_data': USER_DATA, + 'openstack/latest/vendor_data.json': json.dumps(VENDOR_DATA), +} +EC2_FILES = { + 'latest/user-data': USER_DATA, +} + + +def _register_uris(version): + + def match_ec2_url(uri, headers): + path = uri.path.lstrip("/") + if path in EC2_FILES: + return (200, headers, EC2_FILES.get(path)) + if path == 'latest/meta-data': + buf = StringIO() + for (k, v) in EC2_META.items(): + if isinstance(v, (list, tuple)): + buf.write("%s/" % (k)) + else: + buf.write("%s" % (k)) + buf.write("\n") + return (200, headers, buf.getvalue()) + if path.startswith('latest/meta-data'): + value = None + pieces = path.split("/") + if path.endswith("/"): + pieces = pieces[2:-1] + value = util.get_cfg_by_path(EC2_META, pieces) + else: + pieces = pieces[2:] + value = util.get_cfg_by_path(EC2_META, pieces) + if value is not None: + return (200, headers, str(value)) + return (404, headers, '') + + def get_request_callback(method, uri, headers): + uri = urlparse(uri) + path = uri.path.lstrip("/") + if path in OS_FILES: + return (200, headers, OS_FILES.get(path)) + return match_ec2_url(uri, headers) + + def head_request_callback(method, uri, headers): + uri = urlparse(uri) + path = uri.path.lstrip("/") + for key in OS_FILES.keys(): + if key.startswith(path): + return (200, headers, '') + return (404, headers, '') + + hp.register_uri(hp.GET, re.compile(r'http://169.254.169.254/.*'), + body=get_request_callback) + + hp.register_uri(hp.HEAD, re.compile(r'http://169.254.169.254/.*'), + body=head_request_callback) + + +class TestOpenStackDataSource(helpers.TestCase): + VERSION = 'latest' + + @hp.activate + def test_fetch(self): + _register_uris(self.VERSION) + f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + self.assertEquals(VENDOR_DATA, f.get('vendordata')) + self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertEquals(USER_DATA, f.get('userdata')) -- cgit v1.2.3 From 4a0e460f18d8cbf2651286565efec1f00cbb20cd Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Mon, 3 Feb 2014 15:58:43 -0600 Subject: Update yum unittest --- cloudinit/config/cc_set_passwords.py | 3 ++- cloudinit/util.py | 1 - tests/unittests/test_handler/test_handler_yum_add_repo.py | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index b9dc0cc0..ab0ed5ad 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -137,9 +137,10 @@ def handle(_name, cfg, cloud, log, args): util.write_file(ssh_util.DEF_SSHD_CFG, "\n".join(lines)) try: - cmd = cloud.distro.init_cmd + cmd = cloud.distro.init_cmd # Default service cmd.append(cloud.distro.get_option('ssh_svcname', 'ssh')) cmd.append('restart') + cmd = filter(None, cmd) # Remove empty arguments util.subp(cmd) log.debug("Restarted the ssh daemon") except: diff --git a/cloudinit/util.py b/cloudinit/util.py index 54fdad20..3ce54f28 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1483,7 +1483,6 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, logstring=False): if rcs is None: rcs = [0] - args = filter(None, args) # Remove empty arguments try: if not logstring: diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index 8df592f9..ddf0ef9c 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -2,6 +2,7 @@ from cloudinit import helpers from cloudinit import util from cloudinit.config import cc_yum_add_repo +from cloudinit.distros import rhel from tests.unittests import helpers @@ -18,6 +19,8 @@ class TestConfig(helpers.FilesystemMockingTestCase): def setUp(self): super(TestConfig, self).setUp() self.tmp = self.makeDir(prefix="unittest_") + self.cloud = type('', (), {})() + self.cloud.distro = rhel.Distro('test', {}, None) def test_bad_config(self): cfg = { @@ -34,7 +37,7 @@ class TestConfig(helpers.FilesystemMockingTestCase): }, } self.patchUtils(self.tmp) - cc_yum_add_repo.handle('yum_add_repo', cfg, None, LOG, []) + cc_yum_add_repo.handle('yum_add_repo', cfg, self.cloud, LOG, []) self.assertRaises(IOError, util.load_file, "/etc/yum.repos.d/epel_testing.repo") @@ -52,7 +55,7 @@ class TestConfig(helpers.FilesystemMockingTestCase): }, } self.patchUtils(self.tmp) - cc_yum_add_repo.handle('yum_add_repo', cfg, None, LOG, []) + cc_yum_add_repo.handle('yum_add_repo', cfg, self.cloud, LOG, []) contents = util.load_file("/etc/yum.repos.d/epel_testing.repo") contents = configobj.ConfigObj(StringIO(contents)) expected = { -- cgit v1.2.3 From 0efeb26736ddae2967c14a9440088594da32070d Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Mon, 3 Feb 2014 16:53:31 -0600 Subject: Added is_excluded tests and updated gentoo init script --- setup.py | 2 -- sysvinit/gentoo/cloud-init-local | 1 - tests/unittests/test_distros/test_is_excluded.py | 15 +++++++++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 tests/unittests/test_distros/test_is_excluded.py (limited to 'tests') diff --git a/setup.py b/setup.py index b0766970..9118e5f6 100755 --- a/setup.py +++ b/setup.py @@ -39,14 +39,12 @@ def is_f(p): INITSYS_FILES = { 'sysvinit': [f for f in glob('sysvinit/redhat/*') if is_f(f)], 'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)], - 'sysvinit_gentoo': [f for f in glob('sysvinit/gentoo/*') if is_f(f)], 'systemd': [f for f in glob('systemd/*') if is_f(f)], 'upstart': [f for f in glob('upstart/*') if is_f(f)], } INITSYS_ROOTS = { 'sysvinit': '/etc/rc.d/init.d', 'sysvinit_deb': '/etc/init.d', - 'sysvinit_gentoo': '/etc/init.d', 'systemd': '/etc/systemd/system/', 'upstart': '/etc/init/', } diff --git a/sysvinit/gentoo/cloud-init-local b/sysvinit/gentoo/cloud-init-local index 1d22a79d..8c9968d8 100644 --- a/sysvinit/gentoo/cloud-init-local +++ b/sysvinit/gentoo/cloud-init-local @@ -1,7 +1,6 @@ #!/sbin/runscript depend() { - after net # remove after nova-agent fix before cloud-init provide cloud-init-local } diff --git a/tests/unittests/test_distros/test_is_excluded.py b/tests/unittests/test_distros/test_is_excluded.py new file mode 100644 index 00000000..53a4445c --- /dev/null +++ b/tests/unittests/test_distros/test_is_excluded.py @@ -0,0 +1,15 @@ +from cloudinit.distros import gentoo +import unittest + + +class TestIsExcluded(unittest.TestCase): + + def setUp(self): + self.distro = gentoo.Distro('gentoo', {}, None) + self.distro.exclude_modules = ['test-module'] + + def test_is_excluded_success(self): + self.assertEqual(self.distro.is_excluded('test-module'), True) + + def test_is_excluded_fail(self): + self.assertEqual(self.distro.is_excluded('missing'), None) -- cgit v1.2.3 From 805ac503531d27651f0ad4b1a590a488545a0887 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Thu, 6 Feb 2014 09:59:04 -0600 Subject: Removed exclude bits from yum module as its not a default --- cloudinit/config/cc_yum_add_repo.py | 2 -- cloudinit/distros/gentoo.py | 2 +- tests/unittests/test_handler/test_handler_yum_add_repo.py | 7 ++----- 3 files changed, 3 insertions(+), 8 deletions(-) (limited to 'tests') diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index f63e3e08..5c273825 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -58,8 +58,6 @@ def _format_repository_config(repo_id, repo_config): def handle(name, cfg, _cloud, log, _args): - if _cloud.distro.is_excluded(name): - return repos = cfg.get('yum_repos') if not repos: log.debug(("Skipping module named %s," diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index e8778e15..0087908a 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -181,4 +181,4 @@ class Distro(distros.Distro): def update_package_sources(self): self._runner.run("update-sources", self.package_command, - ["-u", "world", "--quiet"], freq=PER_INSTANCE) + ["-u", "world"], freq=PER_INSTANCE) diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index ddf0ef9c..8df592f9 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -2,7 +2,6 @@ from cloudinit import helpers from cloudinit import util from cloudinit.config import cc_yum_add_repo -from cloudinit.distros import rhel from tests.unittests import helpers @@ -19,8 +18,6 @@ class TestConfig(helpers.FilesystemMockingTestCase): def setUp(self): super(TestConfig, self).setUp() self.tmp = self.makeDir(prefix="unittest_") - self.cloud = type('', (), {})() - self.cloud.distro = rhel.Distro('test', {}, None) def test_bad_config(self): cfg = { @@ -37,7 +34,7 @@ class TestConfig(helpers.FilesystemMockingTestCase): }, } self.patchUtils(self.tmp) - cc_yum_add_repo.handle('yum_add_repo', cfg, self.cloud, LOG, []) + cc_yum_add_repo.handle('yum_add_repo', cfg, None, LOG, []) self.assertRaises(IOError, util.load_file, "/etc/yum.repos.d/epel_testing.repo") @@ -55,7 +52,7 @@ class TestConfig(helpers.FilesystemMockingTestCase): }, } self.patchUtils(self.tmp) - cc_yum_add_repo.handle('yum_add_repo', cfg, self.cloud, LOG, []) + cc_yum_add_repo.handle('yum_add_repo', cfg, None, LOG, []) contents = util.load_file("/etc/yum.repos.d/epel_testing.repo") contents = configobj.ConfigObj(StringIO(contents)) expected = { -- cgit v1.2.3 From f9582464b157ffb0087b18910af11aa6ed49abcd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 7 Feb 2014 16:34:04 -0800 Subject: Add a bunch of new tests --- cloudinit/sources/DataSourceOpenStack.py | 3 +- tests/unittests/helpers.py | 6 + tests/unittests/test_datasource/test_openstack.py | 197 ++++++++++++++++++++-- 3 files changed, 190 insertions(+), 16 deletions(-) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 44889f4e..621572de 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -121,7 +121,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'Crawl of openstack metadata service', read_metadata_service, args=[self.metadata_address], - kwargs={'ssl_details': self.ssl_details}) + kwargs={'ssl_details': self.ssl_details, + 'version': openstack.OS_LATEST}) except openstack.NonReadable: return False except openstack.BrokenMetadata: diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 5b4f4208..945c1500 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -29,6 +29,12 @@ if (_PY_MAJOR, _PY_MINOR) <= (2, 6): standardMsg = standardMsg % (member, container) self.fail(self._formatMessage(msg, standardMsg)) + def assertIsNone(self, value, msg=None): + if value is not None: + standardMsg = '%r is not None' + standardMsg = standardMsg % (value) + self.fail(self._formatMessage(msg, standardMsg)) + else: class TestCase(unittest.TestCase): pass diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 7d93f1d3..c8cc40db 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -1,12 +1,33 @@ -import re +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import copy import json +import re from StringIO import StringIO from urlparse import urlparse -from tests.unittests import helpers +from tests.unittests import helpers as test_helpers +from cloudinit import helpers +from cloudinit import settings from cloudinit.sources import DataSourceOpenStack as ds from cloudinit.sources.helpers import openstack from cloudinit import util @@ -17,7 +38,7 @@ BASE_URL = "http://169.254.169.254" PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' EC2_META = { 'ami-id': 'ami-00000001', - 'ami-launch-index': 0, + 'ami-launch-index': '0', 'ami-manifest-path': 'FIXME', 'hostname': 'sm-foo-test.novalocal', 'instance-action': 'none', @@ -59,15 +80,17 @@ EC2_FILES = { } -def _register_uris(version): +def _register_uris(version, ec2_files, ec2_meta, os_files): + """Registers a set of url patterns into httpretty that will mimic the + same data returned by the openstack metadata service (and ec2 service).""" def match_ec2_url(uri, headers): path = uri.path.lstrip("/") - if path in EC2_FILES: - return (200, headers, EC2_FILES.get(path)) + if path in ec2_files: + return (200, headers, ec2_files.get(path)) if path == 'latest/meta-data': buf = StringIO() - for (k, v) in EC2_META.items(): + for (k, v) in ec2_meta.items(): if isinstance(v, (list, tuple)): buf.write("%s/" % (k)) else: @@ -79,10 +102,10 @@ def _register_uris(version): pieces = path.split("/") if path.endswith("/"): pieces = pieces[2:-1] - value = util.get_cfg_by_path(EC2_META, pieces) + value = util.get_cfg_by_path(ec2_meta, pieces) else: pieces = pieces[2:] - value = util.get_cfg_by_path(EC2_META, pieces) + value = util.get_cfg_by_path(ec2_meta, pieces) if value is not None: return (200, headers, str(value)) return (404, headers, '') @@ -90,14 +113,14 @@ def _register_uris(version): def get_request_callback(method, uri, headers): uri = urlparse(uri) path = uri.path.lstrip("/") - if path in OS_FILES: - return (200, headers, OS_FILES.get(path)) + if path in os_files: + return (200, headers, os_files.get(path)) return match_ec2_url(uri, headers) def head_request_callback(method, uri, headers): uri = urlparse(uri) path = uri.path.lstrip("/") - for key in OS_FILES.keys(): + for key in os_files.keys(): if key.startswith(path): return (200, headers, '') return (404, headers, '') @@ -109,14 +132,158 @@ def _register_uris(version): body=head_request_callback) -class TestOpenStackDataSource(helpers.TestCase): +class TestOpenStackDataSource(test_helpers.TestCase): VERSION = 'latest' @hp.activate - def test_fetch(self): - _register_uris(self.VERSION) + def test_successful(self): + _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) + f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + self.assertEquals(VENDOR_DATA, f.get('vendordata')) + self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertEquals(2, len(f['files'])) + self.assertEquals(USER_DATA, f.get('userdata')) + self.assertEquals(EC2_META, f.get('ec2-metadata')) + self.assertEquals(2, f.get('version')) + metadata = f['metadata'] + self.assertEquals('nova', metadata.get('availability_zone')) + self.assertEquals('sm-foo-test.novalocal', metadata.get('hostname')) + self.assertEquals('sm-foo-test.novalocal', + metadata.get('local-hostname')) + self.assertEquals('sm-foo-test', metadata.get('name')) + self.assertEquals('b0fa911b-69d4-4476-bbe2-1c92bff6535c', + metadata.get('uuid')) + self.assertEquals('b0fa911b-69d4-4476-bbe2-1c92bff6535c', + metadata.get('instance-id')) + + @hp.activate + def test_no_ec2(self): + _register_uris(self.VERSION, {}, {}, OS_FILES) f = ds.read_metadata_service(BASE_URL, version=self.VERSION) self.assertEquals(VENDOR_DATA, f.get('vendordata')) self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertEquals(USER_DATA, f.get('userdata')) + self.assertEquals({}, f.get('ec2-metadata')) + self.assertEquals(2, f.get('version')) + + @hp.activate + def test_bad_metadata(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files.pop(k, None) + _register_uris(self.VERSION, {}, {}, os_files) + self.assertRaises(openstack.NonReadable, ds.read_metadata_service, + BASE_URL, version=self.VERSION) + + @hp.activate + def test_bad_uuid(self): + os_files = copy.deepcopy(OS_FILES) + os_meta = copy.deepcopy(OSTACK_META) + os_meta.pop('uuid') + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files[k] = json.dumps(os_meta) + _register_uris(self.VERSION, {}, {}, os_files) + self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, + BASE_URL, version=self.VERSION) + + @hp.activate + def test_userdata_empty(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('user_data'): + os_files.pop(k, None) + _register_uris(self.VERSION, {}, {}, os_files) + f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + self.assertEquals(VENDOR_DATA, f.get('vendordata')) + self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertFalse(f.get('userdata')) + + @hp.activate + def test_vendordata_empty(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('vendor_data.json'): + os_files.pop(k, None) + _register_uris(self.VERSION, {}, {}, os_files) + f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertFalse(f.get('vendordata')) + + @hp.activate + def test_vendordata_invalid(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('vendor_data.json'): + os_files[k] = '{' # some invalid json + _register_uris(self.VERSION, {}, {}, os_files) + self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, + BASE_URL, version=self.VERSION) + + @hp.activate + def test_metadata_invalid(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files[k] = '{' # some invalid json + _register_uris(self.VERSION, {}, {}, os_files) + self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, + BASE_URL, version=self.VERSION) + + @hp.activate + def test_datasource(self): + _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) + ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + self.assertIsNone(ds_os.version) + found = ds_os.get_data() + self.assertTrue(found) + self.assertEquals(2, ds_os.version) + md = dict(ds_os.metadata) + md.pop('instance-id', None) + md.pop('local-hostname', None) + self.assertEquals(OSTACK_META, md) + self.assertEquals(EC2_META, ds_os.ec2_metadata) + self.assertEquals(USER_DATA, ds_os.userdata_raw) + self.assertEquals(2, len(ds_os.files)) + self.assertEquals(VENDOR_DATA, ds_os.vendordata_raw) + + @hp.activate + def test_bad_datasource_meta(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files[k] = '{' # some invalid json + _register_uris(self.VERSION, {}, {}, os_files) + ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + self.assertIsNone(ds_os.version) + found = ds_os.get_data() + self.assertFalse(found) + self.assertIsNone(ds_os.version) + + @hp.activate + def test_no_datasource(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files.pop(k) + _register_uris(self.VERSION, {}, {}, os_files) + ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + ds_os.ds_cfg = { + 'max_wait': 0, + 'timeout': 0, + } + self.assertIsNone(ds_os.version) + found = ds_os.get_data() + self.assertFalse(found) + self.assertIsNone(ds_os.version) -- cgit v1.2.3 From 6e40098626531397a339d3a231d821b738e69175 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 7 Feb 2014 16:40:51 -0800 Subject: Adjust detection of python versions and variables exposed --- tests/unittests/helpers.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'tests') diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 945c1500..5bed13cc 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -12,10 +12,27 @@ from cloudinit import util import shutil -# Handle how 2.6 doesn't have the assertIn or assertNotIn +# Used for detecting different python versions +PY2 = False +PY26 = False +PY27 = False +PY3 = False + _PY_VER = sys.version_info _PY_MAJOR, _PY_MINOR = _PY_VER[0:2] if (_PY_MAJOR, _PY_MINOR) <= (2, 6): + if (_PY_MAJOR, _PY_MINOR) == (2, 6): + PY26 = True + if (_PY_MAJOR, _PY_MINOR) >= (2, 0): + PY2 = True +else: + if (_PY_MAJOR, _PY_MINOR) == (2, 7): + PY27 = True + PY2 = True + if (_PY_MAJOR, _PY_MINOR) >= (3, 0): + PY3 = True + +if PY26: # For now add these on, taken from python 2.7 + slightly adjusted class TestCase(unittest.TestCase): def assertIn(self, member, container, msg=None): -- cgit v1.2.3 From 7fb9f75e1bd8b8ef36398c7adeb8d18a4fe9745e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 7 Feb 2014 16:46:32 -0800 Subject: Add test for disabled dsmode --- tests/unittests/test_datasource/test_openstack.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'tests') diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index c8cc40db..3fcf8bc9 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -287,3 +287,26 @@ class TestOpenStackDataSource(test_helpers.TestCase): found = ds_os.get_data() self.assertFalse(found) self.assertIsNone(ds_os.version) + + @hp.activate + def test_disabled_datasource(self): + os_files = copy.deepcopy(OS_FILES) + os_meta = copy.deepcopy(OSTACK_META) + os_meta['meta'] = { + 'dsmode': 'disabled', + } + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files[k] = json.dumps(os_meta) + _register_uris(self.VERSION, {}, {}, os_files) + ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + ds_os.ds_cfg = { + 'max_wait': 0, + 'timeout': 0, + } + self.assertIsNone(ds_os.version) + found = ds_os.get_data() + self.assertFalse(found) + self.assertIsNone(ds_os.version) -- cgit v1.2.3 From 098a74e6207f5d91f515fac63e970375d52795c0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 8 Feb 2014 12:20:33 -0800 Subject: Remove HEAD usage and other small adjustments --- cloudinit/ec2_utils.py | 4 +-- cloudinit/sources/DataSourceOpenStack.py | 1 + cloudinit/sources/helpers/openstack.py | 41 ++++++++++++++--------- cloudinit/url_helper.py | 23 ++----------- tests/unittests/test_datasource/test_openstack.py | 11 ------ 5 files changed, 30 insertions(+), 50 deletions(-) (limited to 'tests') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 91cba20f..a7c9c9ab 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -16,10 +16,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import httplib -from urlparse import (urlparse, urlunparse) - import functools +import httplib import json from cloudinit import log as logging diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 621572de..2c50ed84 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -44,6 +44,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.ssl_details = util.fetch_ssl_details(self.paths) self.version = None self.files = {} + self.ec2_metadata = None def __str__(self): root = sources.DataSource.__str__(self) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 9dbef677..09fb4ad8 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -21,7 +21,6 @@ import abc import base64 import copy -import functools import os from cloudinit import ec2_utils @@ -395,26 +394,38 @@ class ConfigDriveReader(BaseReader): class MetadataReader(BaseReader): def __init__(self, base_url, ssl_details=None, timeout=5, retries=5): super(MetadataReader, self).__init__(base_url) - self._url_reader = functools.partial(url_helper.readurl, - retries=retries, - ssl_details=ssl_details, - timeout=timeout) - self._url_checker = functools.partial(url_helper.existsurl, - ssl_details=ssl_details, - timeout=timeout) - self._ec2_reader = functools.partial(ec2_utils.get_instance_metadata, - ssl_details=ssl_details, - timeout=timeout, - retries=retries) + self.ssl_details = ssl_details + self.timeout = float(timeout) + self.retries = int(retries) def _path_read(self, path): - return str(self._url_reader(path)) + response = url_helper.readurl(path, + retries=self.retries, + ssl_details=self.ssl_details, + timeout=self.timeout) + return response.contents def _path_exists(self, path): - return self._url_checker(path) + + def should_retry_cb(request, cause): + if cause.code >= 400: + return False + return True + + try: + response = url_helper.readurl(path, + retries=self.retries, + ssl_details=self.ssl_details, + timeout=self.timeout, + exception_cb=should_retry_cb) + return response.ok() + except IOError: + return False def _path_join(self, base, *add_ons): return url_helper.combine_url(base, *add_ons) def _read_ec2_metadata(self): - return self._ec2_reader() + return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details, + timeout=self.timeout, + retries=self.retries) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 76a8e29b..a477b185 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -166,35 +166,16 @@ def _get_ssl_args(url, ssl_details): return ssl_args -def existsurl(url, ssl_details=None, timeout=None): - r = _readurl(url, ssl_details=ssl_details, timeout=timeout, - method='HEAD', check_status=False) - return r.ok() - - def readurl(url, data=None, timeout=None, retries=0, sec_between=1, - headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True, exception_cb=None): - return _readurl(url, data=data, timeout=timeout, retries=retries, - sec_between=sec_between, headers=headers, - headers_cb=headers_cb, ssl_details=ssl_details, - check_status=check_status, - allow_redirects=allow_redirects, - exception_cb=exception_cb) - - -def _readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True, exception_cb=None, - method='GET'): + check_status=True, allow_redirects=True, exception_cb=None): url = _cleanurl(url) req_args = { 'url': url, } req_args.update(_get_ssl_args(url, ssl_details)) - scheme = urlparse(url).scheme # pylint: disable=E1101 req_args['allow_redirects'] = allow_redirects - req_args['method'] = method + req_args['method'] = 'GET' if timeout is not None: req_args['timeout'] = max(float(timeout), 0) if data: diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 3fcf8bc9..3a64430a 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -117,20 +117,9 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): return (200, headers, os_files.get(path)) return match_ec2_url(uri, headers) - def head_request_callback(method, uri, headers): - uri = urlparse(uri) - path = uri.path.lstrip("/") - for key in os_files.keys(): - if key.startswith(path): - return (200, headers, '') - return (404, headers, '') - hp.register_uri(hp.GET, re.compile(r'http://169.254.169.254/.*'), body=get_request_callback) - hp.register_uri(hp.HEAD, re.compile(r'http://169.254.169.254/.*'), - body=head_request_callback) - class TestOpenStackDataSource(test_helpers.TestCase): VERSION = 'latest' -- cgit v1.2.3 From 20305aea1eac724069e0bfaaf976ec5caa8c2439 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Feb 2014 14:56:55 -0500 Subject: drop 'is_excluded'. for now, this the mechanism just doesn't seem right. I think i'd rather have the module declare supported distros than have distros declare [un]supported modules. --- cloudinit/config/cc_apt_configure.py | 2 -- cloudinit/config/cc_apt_pipelining.py | 2 -- cloudinit/config/cc_grub_dpkg.py | 2 -- cloudinit/distros/__init__.py | 8 -------- cloudinit/distros/arch.py | 6 ------ cloudinit/distros/gentoo.py | 5 ----- tests/unittests/test_distros/test_is_excluded.py | 15 --------------- 7 files changed, 40 deletions(-) delete mode 100644 tests/unittests/test_distros/test_is_excluded.py (limited to 'tests') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ccb45bb9..29c13a3d 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -51,8 +51,6 @@ EXPORT_GPG_KEYID = """ def handle(name, cfg, cloud, log, _args): - if cloud.distro.is_excluded(name): - return release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) if not mirrors or "primary" not in mirrors: diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index bd180e82..503a1485 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -35,8 +35,6 @@ APT_PIPE_TPL = ("//Written by cloud-init per 'apt_pipelining'\n" def handle(_name, cfg, _cloud, log, _args): - if _cloud.distro.is_excluded(_name): - return apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) apt_pipe_value_s = str(apt_pipe_value).lower().strip() diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 03cdd98c..b3ce6fb6 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -29,8 +29,6 @@ def handle(_name, cfg, _cloud, log, _args): idevs = None idevs_empty = None - if _cloud.distro.is_excluded(_name): - return if "grub-dpkg" in cfg: idevs = util.get_cfg_option_str(cfg["grub-dpkg"], "grub-pc/install_devices", None) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 8fc0da9f..55d6bcbc 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -56,20 +56,12 @@ class Distro(object): hostname_conf_fn = "/etc/hostname" tz_zone_dir = "/usr/share/zoneinfo" init_cmd = ['service'] # systemctl, service etc - exclude_modules = [] def __init__(self, name, cfg, paths): self._paths = paths self._cfg = cfg self.name = name - def is_excluded(self, name): - if name in self.exclude_modules: - distro = getattr(self, name, None) or getattr(self, 'osfamily') - LOG.debug(("Skipping module named %s, distro %s excluded"), name, - distro) - return True - @abc.abstractmethod def install_packages(self, pkglist): raise NotImplementedError() diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 27dcaa88..310c3dff 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -36,12 +36,6 @@ class Distro(distros.Distro): tz_local_fn = "/etc/localtime" resolve_conf_fn = "/etc/resolv.conf" init_cmd = ['systemctl'] # init scripts - exclude_modules = [ - 'grub-dpkg', - 'apt-configure', - 'apt-pipelining', - 'yum-add-repo', - ] def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 0a95fa23..09f8d8ea 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -34,11 +34,6 @@ class Distro(distros.Distro): tz_conf_fn = "/etc/timezone" tz_local_fn = "/etc/localtime" init_cmd = [''] # init scripts - exclude_modules = [ - 'grub-dpkg', - 'apt-configure', - 'apt-pipelining', - ] def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) diff --git a/tests/unittests/test_distros/test_is_excluded.py b/tests/unittests/test_distros/test_is_excluded.py deleted file mode 100644 index 53a4445c..00000000 --- a/tests/unittests/test_distros/test_is_excluded.py +++ /dev/null @@ -1,15 +0,0 @@ -from cloudinit.distros import gentoo -import unittest - - -class TestIsExcluded(unittest.TestCase): - - def setUp(self): - self.distro = gentoo.Distro('gentoo', {}, None) - self.distro.exclude_modules = ['test-module'] - - def test_is_excluded_success(self): - self.assertEqual(self.distro.is_excluded('test-module'), True) - - def test_is_excluded_fail(self): - self.assertEqual(self.distro.is_excluded('missing'), None) -- cgit v1.2.3 From 046352fc69e94f4c2f2d209ff5638fc455ff4b97 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Thu, 13 Feb 2014 21:24:41 +0000 Subject: GCE: add unit tests, user-data support and few other fixes --- cloudinit/sources/DataSourceGCE.py | 49 ++++++------ tests/unittests/test_datasource/test_gce.py | 112 ++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 24 deletions(-) create mode 100644 tests/unittests/test_datasource/test_gce.py (limited to 'tests') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 95a410ba..2d5ccad5 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -23,7 +23,7 @@ from cloudinit import url_helper LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' + 'metadata_url': 'http://169.254.169.254/computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') @@ -31,7 +31,7 @@ REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') class DataSourceGCE(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.metadata = {} + self.metadata = dict() self.ds_cfg = util.mergemanydict([ util.get_cfg_by_path(sys_cfg, ["datasource", "GCE"], {}), BUILTIN_DS_CONFIG]) @@ -59,33 +59,31 @@ class DataSourceGCE(sources.DataSource): 'user-data': self.metadata_address + 'instance/attributes/user-data', } - # if we cannot resolve the metadata server, then no point in trying - if not util.is_resolvable(self.metadata_address): - LOG.debug("%s is not resolvable", self.metadata_address) + # try to connect to metadata service or fail otherwise + try: + url_helper.readurl(url=url_map['instance-id'], headers=headers) + except url_helper.UrlError as e: + LOG.debug('Failed to connect to metadata service. Reason: {0}'.format( + e)) return False + # iterate over url_map keys to get metadata items for mkey in url_map.iterkeys(): try: - resp = url_helper.readurl(url=url_map[mkey], headers=headers, - retries=0) - except IOError: - return False - if resp.ok(): - if mkey == 'public-keys': - pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] - self.metadata[mkey] = pub_keys + resp = url_helper.readurl( + url=url_map[mkey], headers=headers) + if resp.code == 200: + if mkey == 'public-keys': + pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] + self.metadata[mkey] = pub_keys + else: + self.metadata[mkey] = resp.contents else: - self.metadata[mkey] = resp.contents - else: - if mkey in REQUIRED_FIELDS: - LOG.warn("required metadata '%s' not found in metadata", - url_map[mkey]) - return False - + self.metadata[mkey] = None + except url_helper.UrlError as e: self.metadata[mkey] = None - return False - - self.user_data_raw = self.metadata['user-data'] + LOG.warn('Failed to get {0} metadata item. Reason {1}.'.format( + mkey, e)) return True @property @@ -102,9 +100,12 @@ class DataSourceGCE(sources.DataSource): def get_hostname(self, fqdn=False): return self.metadata['local-hostname'] + def get_userdata_raw(self): + return self.metadata['user-data'] + @property def availability_zone(self): - return self.metadata['instance-zone'] + return self.metadata['availability-zone'] # Used to match classes to dependencies datasources = [ diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py new file mode 100644 index 00000000..0c58be52 --- /dev/null +++ b/tests/unittests/test_datasource/test_gce.py @@ -0,0 +1,112 @@ +# +# Copyright (C) 2014 Vaidas Jablonskis +# +# Author: Vaidas Jablonskis +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import unittest +import httpretty +import re + +from urlparse import urlparse + +from cloudinit import settings +from cloudinit import helpers +from cloudinit.sources import DataSourceGCE + +GCE_META = { + 'instance/id': '123', + 'instance/zone': 'foo/bar', + 'project/attributes/sshKeys': 'user:ssh-rsa AA2..+aRD0fyVw== root@server', + 'instance/hostname': 'server.project-name.local', + 'instance/attributes/user-data': '/bin/echo foo\n', +} + +GCE_META_PARTIAL = { + 'instance/id': '123', + 'instance/hostname': 'server.project-name.local', +} + +HEADERS = {'X-Google-Metadata-Request': 'True'} +MD_URL_RE = re.compile(r'http://169.254.169.254/computeMetadata/v1/.*') + + +def _request_callback(method, uri, headers): + url_path = urlparse(uri).path + if url_path.startswith('/computeMetadata/v1/'): + path = url_path.split('/computeMetadata/v1/')[1:][0] + else: + path = None + if path in GCE_META: + #return (200, headers, GCE_META.get(path)) + return (200, headers, GCE_META.get(path)) + else: + return (404, headers, '') + + +class TestDataSourceGCE(unittest.TestCase): + + def setUp(self): + self.ds = DataSourceGCE.DataSourceGCE( + settings.CFG_BUILTIN, None, + helpers.Paths({})) + + @httpretty.activate + def test_connection(self): + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + + success = self.ds.get_data() + self.assertTrue(success) + + req_header = httpretty.last_request().headers + self.assertDictContainsSubset(HEADERS, req_header) + + @httpretty.activate + def test_metadata(self): + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + self.ds.get_data() + + self.assertEqual(GCE_META.get('instance/hostname'), + self.ds.get_hostname()) + + self.assertEqual(GCE_META.get('instance/id'), + self.ds.get_instance_id()) + + self.assertEqual(GCE_META.get('instance/zone'), + self.ds.availability_zone) + + self.assertEqual(GCE_META.get('instance/attributes/user-data'), + self.ds.get_userdata_raw()) + + # we expect a list of public ssh keys with user names stripped + self.assertEqual(['ssh-rsa AA2..+aRD0fyVw== root@server'], + self.ds.get_public_ssh_keys()) + + # test partial metadata (missing user-data in particular) + @httpretty.activate + def test_metadata_partial(self): + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + self.ds.get_data() + + self.assertEqual(GCE_META_PARTIAL.get('instance/id'), + self.ds.get_instance_id()) + + self.assertEqual(GCE_META_PARTIAL.get('instance/hostname'), + self.ds.get_hostname()) -- cgit v1.2.3 From f40bdb4869567124dbc48feaa0574cc83df26e3a Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Thu, 13 Feb 2014 22:03:12 +0000 Subject: GCE: use dns name instead of IP address --- cloudinit/sources/DataSourceGCE.py | 11 ++++------- tests/unittests/test_datasource/test_gce.py | 3 +-- 2 files changed, 5 insertions(+), 9 deletions(-) (limited to 'tests') diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 2d5ccad5..e2be15b0 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -23,7 +23,7 @@ from cloudinit import url_helper LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://169.254.169.254/computeMetadata/v1/' + 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') @@ -59,12 +59,9 @@ class DataSourceGCE(sources.DataSource): 'user-data': self.metadata_address + 'instance/attributes/user-data', } - # try to connect to metadata service or fail otherwise - try: - url_helper.readurl(url=url_map['instance-id'], headers=headers) - except url_helper.UrlError as e: - LOG.debug('Failed to connect to metadata service. Reason: {0}'.format( - e)) + # if we cannot resolve the metadata server, then no point in trying + if not util.is_resolvable(self.metadata_address): + LOG.debug('{0} is not resolvable'.format(self.metadata_address)) return False # iterate over url_map keys to get metadata items diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 0c58be52..d91bd531 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -39,7 +39,7 @@ GCE_META_PARTIAL = { } HEADERS = {'X-Google-Metadata-Request': 'True'} -MD_URL_RE = re.compile(r'http://169.254.169.254/computeMetadata/v1/.*') +MD_URL_RE = re.compile(r'http://metadata.google.internal./computeMetadata/v1/.*') def _request_callback(method, uri, headers): @@ -49,7 +49,6 @@ def _request_callback(method, uri, headers): else: path = None if path in GCE_META: - #return (200, headers, GCE_META.get(path)) return (200, headers, GCE_META.get(path)) else: return (404, headers, '') -- cgit v1.2.3