From d723c17b7abe5263d879d748dade763034cce476 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 22 Aug 2014 11:40:25 -0400 Subject: ec2_utils.py: get_instance_metadata explicitly add trailing / This change works around a bug in CloudStack's EC2 api implementation. That is filed upstream at [1]. The fix is safe for openstack and EC2 use cases as well. EC2 and OpenStacks' EC2 metadata service both return a list with access to either of: /latest/meta-data /latest/meta-data/ Additionally, the responses explicitly contain a trailing '/' for items that have a child. The ec2_utils code then just re-uses the trailng / there. Thus, only the top level request for 'meta-data/' needs the explicit fix. This also changes test cases. Those test cases failed without fixing them. If ever this regressed, those would fail again. -- [1] https://issues.apache.org/jira/browse/CLOUDSTACK-7405 LP: #1356855 --- tests/unittests/test_datasource/test_openstack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index c088bb55..f43cbec8 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -88,7 +88,7 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): path = uri.path.lstrip("/") if path in ec2_files: return (200, headers, ec2_files.get(path)) - if path == 'latest/meta-data': + if path == 'latest/meta-data/': buf = StringIO() for (k, v) in ec2_meta.items(): if isinstance(v, (list, tuple)): @@ -97,7 +97,7 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): buf.write("%s" % (k)) buf.write("\n") return (200, headers, buf.getvalue()) - if path.startswith('latest/meta-data'): + if path.startswith('latest/meta-data/'): value = None pieces = path.split("/") if path.endswith("/"): -- cgit v1.2.3 From b3216b56f3fea3259c290faa2dd496215b625904 Mon Sep 17 00:00:00 2001 From: Jay Faulkner Date: Tue, 26 Aug 2014 11:50:11 -0700 Subject: fix(pep8): Fix various pep8 violations and version-lock pep8 Fixed all complaints from running "make pep8". Also version locked pep8 in test-requirements.txt to ensure that pep8 requirements don't change without an explicit commit. --- cloudinit/config/cc_apt_configure.py | 2 +- cloudinit/config/cc_byobu.py | 2 +- cloudinit/config/cc_chef.py | 3 +- cloudinit/config/cc_disk_setup.py | 4 +-- cloudinit/config/cc_grub_dpkg.py | 3 +- cloudinit/config/cc_phone_home.py | 2 +- cloudinit/config/cc_resizefs.py | 2 +- cloudinit/config/cc_rightscale_userdata.py | 34 +++++++++++----------- cloudinit/config/cc_rsyslog.py | 4 +-- cloudinit/config/cc_ssh.py | 2 +- cloudinit/config/cc_ssh_authkey_fingerprints.py | 2 +- cloudinit/config/cc_yum_add_repo.py | 4 +-- cloudinit/distros/__init__.py | 4 +-- cloudinit/mergers/m_list.py | 2 +- cloudinit/sources/DataSourceAzure.py | 4 +-- cloudinit/sources/DataSourceNoCloud.py | 2 +- cloudinit/sources/DataSourceOVF.py | 4 +-- cloudinit/sources/DataSourceSmartOS.py | 32 ++++++++++---------- cloudinit/sources/__init__.py | 2 +- cloudinit/stages.py | 2 +- cloudinit/url_helper.py | 4 +-- cloudinit/util.py | 6 ++-- test-requirements.txt | 2 +- tests/unittests/test_datasource/test_azure.py | 2 +- tests/unittests/test_datasource/test_nocloud.py | 4 +-- tests/unittests/test_distros/test_generic.py | 2 +- .../test_handler/test_handler_growpart.py | 8 ++--- .../test_handler/test_handler_power_state.py | 2 +- .../test_handler/test_handler_yum_add_repo.py | 2 +- tools/hacking.py | 2 +- 30 files changed, 76 insertions(+), 74 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 29c13a3d..f10b76a3 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -235,7 +235,7 @@ def find_apt_mirror_info(cloud, cfg): mirror = util.search_for_mirror(search) if (not mirror and - util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)): + util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)): mydom = "" doms = [] diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 92d428b7..ef0ce7ab 100644 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -43,7 +43,7 @@ def handle(name, cfg, cloud, log, args): valid = ("enable-user", "enable-system", "enable", "disable-user", "disable-system", "disable") - if not value in valid: + if value not in valid: log.warn("Unknown value %s for byobu_by_default", value) mod_user = value.endswith("-user") diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 727769cd..806deed9 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -87,7 +87,8 @@ def handle(name, cfg, cloud, log, _args): # If chef is not installed, we install chef based on 'install_type' if (not os.path.isfile('/usr/bin/chef-client') or - util.get_cfg_option_bool(chef_cfg, 'force_install', default=False)): + util.get_cfg_option_bool(chef_cfg, + 'force_install', default=False)): install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 0b970e4e..a5209268 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -271,7 +271,7 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None, return ('/dev/%s' % d['name'], False) if (d['fstype'] == fs_type and - ((label_match and d['label'] == label) or not label_match)): + ((label_match and d['label'] == label) or not label_match)): # If we find a matching device, we return that return ('/dev/%s' % d['name'], True) @@ -447,7 +447,7 @@ def get_partition_mbr_layout(size, layout): return "0," if ((len(layout) == 0 and isinstance(layout, list)) or - not isinstance(layout, list)): + not isinstance(layout, list)): raise Exception("Partition layout is invalid") last_part_num = len(layout) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index b3ce6fb6..85716a91 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -36,7 +36,8 @@ def handle(_name, cfg, _cloud, log, _args): "grub-pc/install_devices_empty", None) if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): + (os.path.exists("/dev/xvda1") + and not os.path.exists("/dev/xvda"))): if idevs is None: idevs = "" if idevs_empty is None: diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 2e058ccd..5bc68b83 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -47,7 +47,7 @@ def handle(name, cfg, cloud, log, args): if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: - if not 'phone_home' in cfg: + if 'phone_home' not in cfg: log.debug(("Skipping module named %s, " "no 'phone_home' configuration found"), name) return diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index be406034..667d5977 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -105,7 +105,7 @@ def handle(name, cfg, _cloud, log, args): container = util.is_container() if (devpth == "/dev/root" and not os.path.exists(devpth) and - not container): + not container): devpth = rootdev_from_cmdline(util.get_cmdline()) if devpth is None: log.warn("Unable to find device '/dev/root'") diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index c771728d..7d2ec10a 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -18,22 +18,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -## -## The purpose of this script is to allow cloud-init to consume -## rightscale style userdata. rightscale user data is key-value pairs -## in a url-query-string like format. -## -## for cloud-init support, there will be a key named -## 'CLOUD_INIT_REMOTE_HOOK'. -## -## This cloud-config module will -## - read the blob of data from raw user data, and parse it as key/value -## - for each key that is found, download the content to -## the local instance/scripts directory and set them executable. -## - the files in that directory will be run by the user-scripts module -## Therefore, this must run before that. -## -## +# +# The purpose of this script is to allow cloud-init to consume +# rightscale style userdata. rightscale user data is key-value pairs +# in a url-query-string like format. +# +# for cloud-init support, there will be a key named +# 'CLOUD_INIT_REMOTE_HOOK'. +# +# This cloud-config module will +# - read the blob of data from raw user data, and parse it as key/value +# - for each key that is found, download the content to +# the local instance/scripts directory and set them executable. +# - the files in that directory will be run by the user-scripts module +# Therefore, this must run before that. +# +# import os @@ -58,7 +58,7 @@ def handle(name, _cfg, cloud, log, _args): try: mdict = parse_qs(ud) - if not mdict or not MY_HOOKNAME in mdict: + if mdict or MY_HOOKNAME not in mdict: log.debug(("Skipping module %s, " "did not find %s in parsed" " raw userdata"), name, MY_HOOKNAME) diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 0c2c6880..57486edc 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -35,7 +35,7 @@ def handle(name, cfg, cloud, log, _args): # *.* @@syslogd.example.com # process 'rsyslog' - if not 'rsyslog' in cfg: + if 'rsyslog' not in cfg: log.debug(("Skipping module named %s," " no 'rsyslog' key in configuration"), name) return @@ -46,7 +46,7 @@ def handle(name, cfg, cloud, log, _args): files = [] for i, ent in enumerate(cfg['rsyslog']): if isinstance(ent, dict): - if not "content" in ent: + if "content" not in ent: log.warn("No 'content' entry in config entry %s", i + 1) continue content = ent['content'] diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 64a5e3cb..4c76581c 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -75,7 +75,7 @@ def handle(_name, cfg, cloud, log, _args): util.write_file(tgt_fn, val, tgt_perms) for (priv, pub) in PRIV_2_PUB.iteritems(): - if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: + if pub in cfg['ssh_keys'] or priv not in cfg['ssh_keys']: continue pair = (KEY_2_FILE[priv][0], KEY_2_FILE[pub][0]) cmd = ['sh', '-xc', KEY_GEN_TPL % pair] diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 7da62589..51580633 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -55,7 +55,7 @@ def _gen_fingerprint(b64_text, hash_meth='md5'): def _is_printable_key(entry): if any([entry.keytype, entry.base64, entry.comment, entry.options]): if (entry.keytype and - entry.keytype.lower().strip() in ['ssh-dss', 'ssh-rsa']): + entry.keytype.lower().strip() in ['ssh-dss', 'ssh-rsa']): return True return False diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 5c273825..0d836f28 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -42,7 +42,7 @@ def _format_repo_value(val): return val -## TODO(harlowja): move to distro? +# TODO(harlowja): move to distro? # See man yum.conf def _format_repository_config(repo_id, repo_config): to_be = configobj.ConfigObj() @@ -89,7 +89,7 @@ def handle(name, cfg, _cloud, log, _args): repo_config = n_repo_config missing_required = 0 for req_field in ['baseurl']: - if not req_field in repo_config: + if req_field not in repo_config: log.warn(("Repository %s does not contain a %s" " configuration 'required' entry"), repo_id, req_field) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 1a56dfb3..4b41220e 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -167,7 +167,7 @@ class Distro(object): def expand_osfamily(family_list): distros = [] for family in family_list: - if not family in OSFAMILIES: + if family not in OSFAMILIES: raise ValueError("No distibutions found for osfamily %s" % (family)) distros.extend(OSFAMILIES[family]) @@ -218,7 +218,7 @@ class Distro(object): fn) if (sys_hostname and prev_hostname and - sys_hostname != prev_hostname): + sys_hostname != prev_hostname): LOG.debug("%s differs from %s, assuming user maintained hostname.", prev_hostname_fn, sys_fn) diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index 62999b4e..3b87b0fc 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -53,7 +53,7 @@ class Merger(object): def _on_list(self, value, merge_with): if (self._method == 'replace' and - not isinstance(merge_with, (tuple, list))): + not isinstance(merge_with, (tuple, list))): return merge_with # Ok we now know that what we are merging with is a list or tuple. diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index bd75e6d8..09bc196d 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -452,7 +452,7 @@ def load_azure_ovf_pubkeys(sshnode): continue if (len(child.childNodes) != 1 or - child.childNodes[0].nodeType != text_node): + child.childNodes[0].nodeType != text_node): continue cur[name] = child.childNodes[0].wholeText.strip() @@ -521,7 +521,7 @@ def read_azure_ovf(contents): simple = False value = "" if (len(child.childNodes) == 1 and - child.childNodes[0].nodeType == dom.TEXT_NODE): + child.childNodes[0].nodeType == dom.TEXT_NODE): simple = True value = child.childNodes[0].wholeText diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index a315aae0..c26a645c 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -181,7 +181,7 @@ class DataSourceNoCloud(sources.DataSource): # and the source of the seed was self.dsmode # ('local' for NoCloud, 'net' for NoCloudNet') if ('network-interfaces' in mydata['meta-data'] and - (self.dsmode in ("local", seeded_interfaces))): + (self.dsmode in ("local", seeded_interfaces))): LOG.debug("Updating network interfaces from %s", self) self.distro.apply_network( mydata['meta-data']['network-interfaces']) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 77b43e17..2f53c1ba 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -107,7 +107,7 @@ class DataSourceOVF(sources.DataSource): return True def get_public_ssh_keys(self): - if not 'public-keys' in self.metadata: + if 'public-keys' not in self.metadata: return [] pks = self.metadata['public-keys'] if isinstance(pks, (list)): @@ -205,7 +205,7 @@ def transport_iso9660(require_iso=True): fullp = os.path.join("/dev/", dev) if (fullp in mounts or - not cdmatch.match(dev) or os.path.isdir(fullp)): + not cdmatch.match(dev) or os.path.isdir(fullp)): continue try: diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 65ec0339..2733a2f6 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -41,7 +41,7 @@ import serial LOG = logging.getLogger(__name__) SMARTOS_ATTRIB_MAP = { - #Cloud-init Key : (SmartOS Key, Strip line endings) + # Cloud-init Key : (SmartOS Key, Strip line endings) 'local-hostname': ('hostname', True), 'public-keys': ('root_authorized_keys', True), 'user-script': ('user-script', False), @@ -96,21 +96,21 @@ BUILTIN_CLOUD_CONFIG = { 'device': 'ephemeral0'}], } -## builtin vendor-data is a boothook that writes a script into -## /var/lib/cloud/scripts/per-boot. *That* script then handles -## executing the 'operator-script' and 'user-script' files -## that cloud-init writes into /var/lib/cloud/instance/data/ -## if they exist. -## -## This is all very indirect, but its done like this so that at -## some point in the future, perhaps cloud-init wouldn't do it at -## all, but rather the vendor actually provide vendor-data that accomplished -## their desires. (That is the point of vendor-data). -## -## cloud-init does cheat a bit, and write the operator-script and user-script -## itself. It could have the vendor-script do that, but it seems better -## to not require the image to contain a tool (mdata-get) to read those -## keys when we have a perfectly good one inside cloud-init. +# builtin vendor-data is a boothook that writes a script into +# /var/lib/cloud/scripts/per-boot. *That* script then handles +# executing the 'operator-script' and 'user-script' files +# that cloud-init writes into /var/lib/cloud/instance/data/ +# if they exist. +# +# This is all very indirect, but its done like this so that at +# some point in the future, perhaps cloud-init wouldn't do it at +# all, but rather the vendor actually provide vendor-data that accomplished +# their desires. (That is the point of vendor-data). +# +# cloud-init does cheat a bit, and write the operator-script and user-script +# itself. It could have the vendor-script do that, but it seems better +# to not require the image to contain a tool (mdata-get) to read those +# keys when we have a perfectly good one inside cloud-init. BUILTIN_VENDOR_DATA = """\ #cloud-boothook #!/bin/sh diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index fef4d460..7d52a2e6 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -166,7 +166,7 @@ class DataSource(object): defhost = "localhost" domain = defdomain - if not self.metadata or not 'local-hostname' in self.metadata: + if self.metadata or 'local-hostname' not in self.metadata: # this is somewhat questionable really. # the cloud datasource was asked for a hostname # and didn't have one. raising error might be more appropriate diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 9e071fc4..d29d480a 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -642,7 +642,7 @@ class Modules(object): # Try the modules frequency, otherwise fallback to a known one if not freq: freq = mod.frequency - if not freq in FREQUENCIES: + if freq not in FREQUENCIES: freq = PER_INSTANCE LOG.debug("Running module %s (%s) with frequency %s", name, mod, freq) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 4a83169a..73c1fa4e 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -237,8 +237,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, return UrlResponse(r) except exceptions.RequestException as e: if (isinstance(e, (exceptions.HTTPError)) - and hasattr(e, 'response') # This appeared in v 0.10.8 - and hasattr(e.response, 'status_code')): + and hasattr(e, 'response') # This appeared in v 0.10.8 + and hasattr(e.response, 'status_code')): excps.append(UrlError(e, code=e.response.status_code, headers=e.response.headers)) else: diff --git a/cloudinit/util.py b/cloudinit/util.py index bc681f4a..0821901a 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -423,7 +423,7 @@ def get_cfg_option_list(yobj, key, default=None): @return: The configuration option as a list of strings or default if key is not found. """ - if not key in yobj: + if key not in yobj: return default if yobj[key] is None: return [] @@ -1148,7 +1148,7 @@ def chownbyname(fname, user=None, group=None): # this returns the specific 'mode' entry, cleanly formatted, with value def get_output_cfg(cfg, mode): ret = [None, None] - if not cfg or not 'output' in cfg: + if cfg or 'output' not in cfg: return ret outcfg = cfg['output'] @@ -1745,7 +1745,7 @@ def parse_mount_info(path, mountinfo_lines, log=LOG): # Ignore mount points higher than an already seen mount # point. if (match_mount_point_elements is not None and - len(match_mount_point_elements) > len(mount_point_elements)): + len(match_mount_point_elements) > len(mount_point_elements)): continue # Find the '-' which terminates a list of optional columns to diff --git a/test-requirements.txt b/test-requirements.txt index 4be0211d..47d45d47 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ httpretty>=0.7.1 mocker nose -pep8 +pep8==1.5.7 pyflakes pylint diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 88c82d5e..e992a006 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -235,7 +235,7 @@ class TestAzureDataSource(MockerTestCase): self.assertEqual(dsrc.userdata_raw, mydata) def test_no_datasource_expected(self): - #no source should be found if no seed_dir and no devs + # no source should be found if no seed_dir and no devs data = {} dsrc = self._get_ds({}) ret = dsrc.get_data() diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 14274562..8bcc026c 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -50,7 +50,7 @@ class TestNoCloudDataSource(MockerTestCase): self.assertTrue(ret) def test_fs_label(self): - #find_devs_with should not be called ff fs_label is None + # find_devs_with should not be called ff fs_label is None ds = DataSourceNoCloud.DataSourceNoCloud class PsuedoException(Exception): @@ -74,7 +74,7 @@ class TestNoCloudDataSource(MockerTestCase): self.assertFalse(ret) def test_no_datasource_expected(self): - #no source should be found if no cmdline, config, and fs_label=None + # no source should be found if no cmdline, config, and fs_label=None sys_cfg = {'datasource': {'NoCloud': {'fs_label': None}}} ds = DataSourceNoCloud.DataSourceNoCloud diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index c24c790e..a972568f 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -193,7 +193,7 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): 'security': 'http://security-mirror2-intel'}) -#def _get_package_mirror_info(mirror_info, availability_zone=None, +# def _get_package_mirror_info(mirror_info, availability_zone=None, # mirror_filter=util.search_for_mirror): diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index f6dc4521..fa624197 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -53,7 +53,7 @@ class TestDisabled(MockerTestCase): self.handle = cc_growpart.handle def test_mode_off(self): - #Test that nothing is done if mode is off. + # Test that nothing is done if mode is off. # this really only verifies that resizer_factory isn't called config = {'growpart': {'mode': 'off'}} @@ -109,7 +109,7 @@ class TestConfig(MockerTestCase): self.assertTrue(isinstance(ret, cc_growpart.ResizeGrowPart)) def test_handle_with_no_growpart_entry(self): - #if no 'growpart' entry in config, then mode=auto should be used + # if no 'growpart' entry in config, then mode=auto should be used myresizer = object() @@ -141,7 +141,7 @@ class TestResize(MockerTestCase): self.mocker.order() def test_simple_devices(self): - #test simple device list + # test simple device list # this patches out devent2dev, os.stat, and device_part_info # so in the end, doesn't test a lot devs = ["/dev/XXda1", "/dev/YYda2"] @@ -187,7 +187,7 @@ class TestResize(MockerTestCase): find("/dev/YYda2", resized)[1]) self.assertEqual(cc_growpart.RESIZE.SKIPPED, find(enoent[0], resized)[1]) - #self.assertEqual(resize_calls, + # self.assertEqual(resize_calls, # [("/dev/XXda", "1", "/dev/XXda1"), # ("/dev/YYda", "2", "/dev/YYda2")]) finally: diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 4b7b2112..2f86b8f8 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -67,7 +67,7 @@ def check_lps_ret(psc_return, mode=None): cmd = psc_return[0] timeout = psc_return[1] - if not 'shutdown' in psc_return[0][0]: + if 'shutdown' not in psc_return[0][0]: errs.append("string 'shutdown' not in cmd") if mode is not None: diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index 156441c7..21b89c34 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -24,7 +24,7 @@ class TestConfig(helpers.FilesystemMockingTestCase): 'epel-testing': { 'name': 'Extra Packages for Enterprise Linux 5 - Testing', # Missing this should cause the repo not to be written - #'baseurl': 'http://blah.org/pub/epel/testing/5/$basearch', + # 'baseurl': 'http://blah.org/pub/epel/testing/5/$basearch', 'enabled': False, 'gpgcheck': True, 'gpgkey': 'file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL', diff --git a/tools/hacking.py b/tools/hacking.py index 26a07c53..14bd0cda 100755 --- a/tools/hacking.py +++ b/tools/hacking.py @@ -71,7 +71,7 @@ def cloud_import_alphabetical(physical_line, line_number, lines): # with or without "as y" length = [2, 4] if (len(split_line) in length and len(split_previous) in length and - split_line[0] == "import" and split_previous[0] == "import"): + split_line[0] == "import" and split_previous[0] == "import"): if split_line[1] < split_previous[1]: return (0, "N306: imports not in alphabetical order (%s, %s)" % (split_previous[1], split_line[1])) -- cgit v1.2.3 From d701035265c765bc42cb3bc358f2bfd0b41f484b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 27 Aug 2014 12:30:23 -0700 Subject: Fixed more of the slowness around fetching and retrying --- cloudinit/sources/helpers/openstack.py | 112 +++++++++++++--------- tests/unittests/test_datasource/test_openstack.py | 23 ++++- 2 files changed, 84 insertions(+), 51 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 3ceec837..2d0fc70e 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -21,6 +21,7 @@ import abc import base64 import copy +import httplib import os from cloudinit import ec2_utils @@ -153,10 +154,31 @@ class BaseReader(object): def _path_read(self, path): pass + @abc.abstractmethod + def _fetch_available_versions(self): + pass + @abc.abstractmethod def _read_ec2_metadata(self): pass + def _find_working_version(self, version): + search_versions = [version] + list(OS_VERSIONS) + available_versions = self._fetch_available_versions() + for potential_version in search_versions: + if not potential_version: + continue + if potential_version not in available_versions: + continue + if potential_version != version: + LOG.debug("Version '%s' not available, attempting to use" + " version '%s' instead", version, + potential_version) + return potential_version + LOG.debug("Version '%s' not available, attempting to use '%s'" + " instead", version, OS_LATEST) + return OS_LATEST + def _read_content_path(self, item): path = item.get('content_path', '').lstrip("/") path_pieces = path.split("/") @@ -166,10 +188,6 @@ class BaseReader(object): path = self._path_join(self.base_path, "openstack", *path_pieces) return self._path_read(path) - @abc.abstractmethod - def _find_working_version(self, version): - pass - def read_v2(self, version=None): """Reads a version 2 formatted location. @@ -290,6 +308,7 @@ class BaseReader(object): class ConfigDriveReader(BaseReader): def __init__(self, base_path): super(ConfigDriveReader, self).__init__(base_path) + self._versions = None def _path_join(self, base, *add_ons): components = [base] + list(add_ons) @@ -298,22 +317,21 @@ class ConfigDriveReader(BaseReader): def _path_read(self, path): return util.load_file(path) - def _find_working_version(self, version): - search_versions = [version] + list(OS_VERSIONS) - for potential_version in search_versions: - if not potential_version: - continue - path = self._path_join(self.base_path, "openstack", - potential_version) - if os.path.exists(path): - if potential_version != version: - LOG.debug("Version '%s' not available, attempting to use" - " version '%s' instead", version, - potential_version) - return potential_version - LOG.debug("Version '%s' not available, attempting to use '%s'" - " instead", version, OS_LATEST) - return OS_LATEST + def _fetch_available_versions(self): + if self._versions is not None: + return self._versions + else: + versions_available = [] + path = self._path_join(self.base_path, 'openstack') + try: + for child in os.listdir(path): + child_path = os.path.join(path, child) + if os.path.isdir(child_path): + versions_available.append(child) + except (OSError, IOError): + pass + self._versions = tuple(versions_available) + return self._versions def _read_ec2_metadata(self): path = self._path_join(self.base_path, @@ -400,40 +418,40 @@ class MetadataReader(BaseReader): self.ssl_details = ssl_details self.timeout = float(timeout) self.retries = int(retries) + self._versions = None - def _find_working_version(self, version): - search_versions = [version] + list(OS_VERSIONS) - version_path = self._path_join(self.base_path, "openstack") - versions_available = [] - try: - versions = self._path_read(version_path) - except IOError as e: - LOG.warn("Unable to read openstack versions from %s due" - " to: %s", version_path, e) + def _fetch_available_versions(self): + if self._versions is not None: + return self._versions else: - for line in versions.splitlines(): - line = line.strip() - if not line: - continue - versions_available.append(line) - for potential_version in search_versions: - if potential_version not in versions_available: - continue - if potential_version != version: - LOG.debug("Version '%s' not available, attempting to use" - " version '%s' instead", version, - potential_version) - return potential_version - LOG.debug("Version '%s' not available, searched for %s (with available" - " versions being %s), attempting to use '%s' instead", - version, search_versions, versions_available, OS_LATEST) - return OS_LATEST + path = self._path_join(self.base_path, "openstack") + versions_available = [] + try: + versions = self._path_read(path) + except IOError as e: + LOG.warn("Unable to read openstack versions from %s due" + " to: %s", path, e) + else: + for line in versions.splitlines(): + line = line.strip() + if not line: + continue + versions_available.append(line) + self._versions = tuple(versions_available) + return self._versions def _path_read(self, path): + + def should_retry(_request_args, cause): + if cause.code == httplib.NOT_FOUND: + return False + return True + response = url_helper.readurl(path, retries=self.retries, ssl_details=self.ssl_details, - timeout=self.timeout) + timeout=self.timeout, + exception_cb=should_retry) return response.contents def _path_join(self, base, *add_ons): diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index f43cbec8..412ae5a4 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -67,8 +67,8 @@ OSTACK_META = { CONTENT_0 = 'This is contents of /etc/foo.cfg\n' CONTENT_1 = '# this is /etc/bar/bar.cfg\n' OS_FILES = { - 'openstack/2012-08-10/meta_data.json': json.dumps(OSTACK_META), - 'openstack/2012-08-10/user_data': USER_DATA, + 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), + 'openstack/latest/user_data': USER_DATA, 'openstack/content/0000': CONTENT_0, 'openstack/content/0001': CONTENT_1, 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), @@ -78,6 +78,9 @@ OS_FILES = { EC2_FILES = { 'latest/user-data': USER_DATA, } +EC2_VERSIONS = [ + 'lastest', +] def _register_uris(version, ec2_files, ec2_meta, os_files): @@ -85,6 +88,9 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): same data returned by the openstack metadata service (and ec2 service).""" def match_ec2_url(uri, headers): + path = uri.path.strip("/") + if len(path) == 0: + return (200, headers, "\n".join(EC2_VERSIONS)) path = uri.path.lstrip("/") if path in ec2_files: return (200, headers, ec2_files.get(path)) @@ -110,11 +116,20 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): return (200, headers, str(value)) return (404, headers, '') - def get_request_callback(method, uri, headers): - uri = urlparse(uri) + def match_os_uri(uri, headers): + path = uri.path.strip("/") + if path == 'openstack': + return (200, headers, "\n".join([openstack.OS_LATEST])) path = uri.path.lstrip("/") if path in os_files: return (200, headers, os_files.get(path)) + return (404, headers, '') + + def get_request_callback(method, uri, headers): + uri = urlparse(uri) + path = uri.path.lstrip("/").split("/") + if path[0] == 'openstack': + return match_os_uri(uri, headers) return match_ec2_url(uri, headers) hp.register_uri(hp.GET, re.compile(r'http://169.254.169.254/.*'), -- cgit v1.2.3 From 419e0caab7e005827485460372c7f0d54ac0e9c9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 27 Aug 2014 17:03:43 -0400 Subject: no functional changes, but some minor changes. --- cloudinit/sources/helpers/openstack.py | 75 ++++++++++------------- tests/unittests/test_datasource/test_openstack.py | 2 +- 2 files changed, 34 insertions(+), 43 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 025a2404..3c6bb6aa 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -162,21 +162,25 @@ class BaseReader(object): pass def _find_working_version(self, version): + try: + versions_available = self._fetch_available_versions(self) + except Exception as e: + LOG.warn("Unable to read openstack versions from %s due to: %s", + self.base_path, e) + versions_available = [] + search_versions = [version] + list(OS_VERSIONS) - available_versions = self._fetch_available_versions() + selected_version = OS_LATEST for potential_version in search_versions: - if not potential_version: + if potential_version not in versions_available: continue - if potential_version not in available_versions: - continue - if potential_version != version: - LOG.debug("Version '%s' not available, attempting to use" - " version '%s' instead", version, - potential_version) - return potential_version - LOG.debug("Version '%s' not available, attempting to use '%s'" - " instead", version, OS_LATEST) - return OS_LATEST + selected_version = potential_version + break + + if selected_version != version: + LOG.warn("Version '%s' not available, attempting to use" + " version '%s' instead", version, selected_version) + return selected_version def _read_content_path(self, item): path = item.get('content_path', '').lstrip("/") @@ -317,20 +321,12 @@ class ConfigDriveReader(BaseReader): return util.load_file(path) def _fetch_available_versions(self): - if self._versions is not None: - return self._versions - else: - versions_available = [] + if self._versions is None: path = self._path_join(self.base_path, 'openstack') - try: - for child in os.listdir(path): - child_path = os.path.join(path, child) - if os.path.isdir(child_path): - versions_available.append(child) - except (OSError, IOError): - pass - self._versions = tuple(versions_available) - return self._versions + found = [d for d in os.listdir(path) + if os.path.isdir(os.path.join(path))] + self._versions = tuple(found) + return self._versions def _read_ec2_metadata(self): path = self._path_join(self.base_path, @@ -420,24 +416,19 @@ class MetadataReader(BaseReader): self._versions = None def _fetch_available_versions(self): + # /openstack/ returns a newline separated list of versions if self._versions is not None: - return self._versions - else: - path = self._path_join(self.base_path, "openstack") - versions_available = [] - try: - versions = self._path_read(path) - except IOError as e: - LOG.warn("Unable to read openstack versions from %s due" - " to: %s", path, e) - else: - for line in versions.splitlines(): - line = line.strip() - if not line: - continue - versions_available.append(line) - self._versions = tuple(versions_available) - return self._versions + return self.os_versions + found = [] + content = self._path_read(version_path) + for line in content.splitlines(): + line = line.strip() + if not line: + continue + found.append(line) + self._versions = tuple(found) + return self._versions + def _path_read(self, path): diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 412ae5a4..530fba20 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -79,7 +79,7 @@ EC2_FILES = { 'latest/user-data': USER_DATA, } EC2_VERSIONS = [ - 'lastest', + 'latest', ] -- cgit v1.2.3 From 44113217719ebee756325b40a5d14045ba8f3a3a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 10 Sep 2014 16:00:00 -0400 Subject: drop version= from readers instead of taking a version that they should look for, the readers now just select the highest supported version. definitely a use case later for having version= but nothing is using it now. --- cloudinit/sources/DataSourceConfigDrive.py | 4 ++-- cloudinit/sources/DataSourceOpenStack.py | 4 ++-- cloudinit/sources/helpers/openstack.py | 23 +++++++---------------- tests/unittests/test_datasource/test_openstack.py | 16 ++++++++-------- 4 files changed, 19 insertions(+), 28 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index b8c16361..a27d07fb 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -167,10 +167,10 @@ def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): return "net" -def read_config_drive(source_dir, version=None): +def read_config_drive(source_dir): reader = openstack.ConfigDriveReader(source_dir) finders = [ - (reader.read_v2, [], {'version': version}), + (reader.read_v2, [], {}), (reader.read_v1, [], {}), ] excps = [] diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index ce8e8364..466de8f4 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -153,9 +153,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): return True -def read_metadata_service(base_url, version=None, ssl_details=None): +def read_metadata_service(base_url, ssl_details=None): reader = openstack.MetadataReader(base_url, ssl_details=ssl_details) - return reader.read_v2(version=version) + return reader.read_v2() # Used to match classes to dependencies diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 3d903a3c..a7dd05df 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -162,7 +162,7 @@ class BaseReader(object): def _read_ec2_metadata(self): pass - def _find_working_version(self, version=None): + def _find_working_version(self): try: versions_available = self._fetch_available_versions() except Exception as e: @@ -173,24 +173,16 @@ class BaseReader(object): # openstack.OS_VERSIONS is stored in chronological order, so # reverse it to check newest first. supported = [v for v in reversed(list(OS_VERSIONS))] - if version is not None: - search_versions = [version] + supported - else: - search_versions = supported - selected_version = OS_LATEST - for potential_version in search_versions: + + for potential_version in supported: if potential_version not in versions_available: continue selected_version = potential_version break - if version is not None and selected_version != version: - LOG.warn("Version '%s' not available, attempting to use " - "version '%s' instead", version, selected_version) - else: - LOG.debug("Selected version '%s' from %s", selected_version, - versions_available) + LOG.debug("Selected version '%s' from %s", selected_version, + versions_available) return selected_version def _read_content_path(self, item): @@ -202,7 +194,7 @@ class BaseReader(object): path = self._path_join(self.base_path, "openstack", *path_pieces) return self._path_read(path) - def read_v2(self, version=None): + def read_v2(self): """Reads a version 2 formatted location. Return a dict with metadata, userdata, ec2-metadata, dsmode, @@ -233,12 +225,11 @@ class BaseReader(object): ) return files - version = self._find_working_version(version) results = { 'userdata': '', 'version': 2, } - data = datafiles(version) + data = datafiles(self._find_working_version()) for (name, (path, required, translator)) in data.iteritems(): path = self._path_join(self.base_path, path) data = None diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 530fba20..6809823e 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -142,7 +142,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): @hp.activate def test_successful(self): _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) - f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + f = ds.read_metadata_service(BASE_URL) self.assertEquals(VENDOR_DATA, f.get('vendordata')) self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -164,7 +164,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): @hp.activate def test_no_ec2(self): _register_uris(self.VERSION, {}, {}, OS_FILES) - f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + f = ds.read_metadata_service(BASE_URL) self.assertEquals(VENDOR_DATA, f.get('vendordata')) self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -180,7 +180,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) self.assertRaises(openstack.NonReadable, ds.read_metadata_service, - BASE_URL, version=self.VERSION) + BASE_URL) @hp.activate def test_bad_uuid(self): @@ -192,7 +192,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): os_files[k] = json.dumps(os_meta) _register_uris(self.VERSION, {}, {}, os_files) self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL, version=self.VERSION) + BASE_URL) @hp.activate def test_userdata_empty(self): @@ -201,7 +201,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('user_data'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + f = ds.read_metadata_service(BASE_URL) self.assertEquals(VENDOR_DATA, f.get('vendordata')) self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -214,7 +214,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('vendor_data.json'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + f = ds.read_metadata_service(BASE_URL) self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertFalse(f.get('vendordata')) @@ -227,7 +227,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): os_files[k] = '{' # some invalid json _register_uris(self.VERSION, {}, {}, os_files) self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL, version=self.VERSION) + BASE_URL) @hp.activate def test_metadata_invalid(self): @@ -237,7 +237,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): os_files[k] = '{' # some invalid json _register_uris(self.VERSION, {}, {}, os_files) self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL, version=self.VERSION) + BASE_URL) @hp.activate def test_datasource(self): -- cgit v1.2.3 From 932c073db79e667623d27174c55e5b16ea439578 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 10 Sep 2014 21:17:40 -0400 Subject: Openstack: Vendor data cleanup For now, this vendor data handling is just added to openstack. However, in an effort to allow sanely handling of multi-part vendor-data that is namespaced, we add openstack.convert_vendordata_json . That basically takes whatever was loaded from vendordata and takes the 'cloud-init' key if it is a dict. This way the author can namespace cloud-init, basically telling it to ignore everything else. --- cloudinit/sources/DataSourceConfigDrive.py | 13 +++++---- cloudinit/sources/DataSourceOpenStack.py | 12 ++++---- cloudinit/sources/helpers/openstack.py | 25 ++++++++++++++++ tests/unittests/test_datasource/test_openstack.py | 35 ++++++++++++++++++++++- 4 files changed, 72 insertions(+), 13 deletions(-) (limited to 'tests/unittests/test_datasource') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index a27d07fb..4e5d90de 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -126,12 +126,13 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): self.version = results['version'] self.files.update(results.get('files', {})) - # If there is no vendordata, set vd to an empty dict instead of None - vd = results.get('vendordata', {}) - # if vendordata includes 'cloud-init', then read that explicitly - # for cloud-init (for namespacing). - if 'cloud-init' in vd: - self.vendordata_raw = vd['cloud-init'] + vd = results.get('vendordata') + self.vendordata_pure = vd + try: + self.vendordata_raw = openstack.convert_vendordata_json(vd) + except ValueError as e: + LOG.warn("Invalid content in vendor-data: %s", e) + self.vendordata_raw = None return True diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 765137c6..469c2e2a 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -140,13 +140,13 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.version = results['version'] self.files.update(results.get('files', {})) - # if vendordata includes 'cloud-init', then read that explicitly - # for cloud-init (for namespacing). vd = results.get('vendordata') - if isinstance(vd, dict) and 'cloud-init' in vd: - self.vendordata_raw = vd['cloud-init'] - else: - self.vendordata_raw = vd + self.vendordata_pure = vd + try: + self.vendordata_raw = openstack.convert_vendordata_json(vd) + except ValueError as e: + LOG.warn("Invalid content in vendor-data: %s", e) + self.vendordata_raw = None return True diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index a7dd05df..9718b0be 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -459,3 +459,28 @@ class MetadataReader(BaseReader): return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details, timeout=self.timeout, retries=self.retries) + + +def convert_vendordata_json(data, recurse=True): + """ data: a loaded json *object* (strings, arrays, dicts). + return something suitable for cloudinit vendordata_raw. + + if data is: + None: return None + string: return string + list: return data + the list is then processed in UserDataProcessor + dict: return convert_vendordata_json(data.get('cloud-init')) + """ + if not data: + return None + if isinstance(data, (str, unicode, basestring)): + return data + if isinstance(data, list): + return copy.deepcopy(data) + if isinstance(data, dict): + if recurse is True: + return convert_vendordata_json(data.get('cloud-init'), + recurse=False) + raise ValueError("vendordata['cloud-init'] cannot be dict") + raise ValueError("Unknown data type for vendordata: %s" % type(data)) diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 6809823e..7b4e651a 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -19,6 +19,7 @@ import copy import json import re +import unittest from StringIO import StringIO @@ -256,7 +257,8 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): self.assertEquals(EC2_META, ds_os.ec2_metadata) self.assertEquals(USER_DATA, ds_os.userdata_raw) self.assertEquals(2, len(ds_os.files)) - self.assertEquals(VENDOR_DATA, ds_os.vendordata_raw) + self.assertEquals(VENDOR_DATA, ds_os.vendordata_pure) + self.assertEquals(ds_os.vendordata_raw, None) @hp.activate def test_bad_datasource_meta(self): @@ -314,3 +316,34 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): found = ds_os.get_data() self.assertFalse(found) self.assertIsNone(ds_os.version) + + +class TestVendorDataLoading(unittest.TestCase): + def cvj(self, data): + return openstack.convert_vendordata_json(data) + + def test_vd_load_none(self): + # non-existant vendor-data should return none + self.assertIsNone(self.cvj(None)) + + def test_vd_load_string(self): + self.assertEqual(self.cvj("foobar"), "foobar") + + def test_vd_load_list(self): + data = [{'foo': 'bar'}, 'mystring', list(['another', 'list'])] + self.assertEqual(self.cvj(data), data) + + def test_vd_load_dict_no_ci(self): + self.assertEqual(self.cvj({'foo': 'bar'}), None) + + def test_vd_load_dict_ci_dict(self): + self.assertRaises(ValueError, self.cvj, + {'foo': 'bar', 'cloud-init': {'x': 1}}) + + def test_vd_load_dict_ci_string(self): + data = {'foo': 'bar', 'cloud-init': 'VENDOR_DATA'} + self.assertEqual(self.cvj(data), data['cloud-init']) + + def test_vd_load_dict_ci_list(self): + data = {'foo': 'bar', 'cloud-init': ['VD_1', 'VD_2']} + self.assertEqual(self.cvj(data), data['cloud-init']) -- cgit v1.2.3