From cdba22dd0fc249124dfb4e578f44898ba044925a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 25 Sep 2012 13:29:07 -0700 Subject: Allow package_mirrors to be non-existent. --- cloudinit/distros/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 549c1612..6df843ad 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -112,7 +112,7 @@ class Distro(object): return arch def _get_arch_package_mirror_info(self, arch=None): - mirror_info = self.get_option("package_mirrors", None) + mirror_info = self.get_option("package_mirrors") or [] if arch == None: arch = self.get_primary_arch() return _get_arch_package_mirror_info(mirror_info, arch) @@ -122,7 +122,6 @@ class Distro(object): # this resolves the package_mirrors config option # down to a single dict of {mirror_name: mirror_url} arch_info = self._get_arch_package_mirror_info(arch) - return _get_package_mirror_info(availability_zone=availability_zone, mirror_info=arch_info) -- cgit v1.2.3 From 332ef83d77853714b88358de301bda30e82aaf62 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 28 Sep 2012 11:06:16 -0700 Subject: Stop the bad habit of using 'or'. --- cloudinit/distros/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 6df843ad..55e4b8db 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -112,7 +112,7 @@ class Distro(object): return arch def _get_arch_package_mirror_info(self, arch=None): - mirror_info = self.get_option("package_mirrors") or [] + mirror_info = self.get_option("package_mirrors", []) if arch == None: arch = self.get_primary_arch() return _get_arch_package_mirror_info(mirror_info, arch) -- cgit v1.2.3 From 46be69003044a7d60d9566dbddd1b7fd93054c8f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 28 Sep 2012 11:08:46 -0700 Subject: Also ensure that if mirror info is not invalid before we start iterating over it (it could be sent is as none). --- cloudinit/distros/__init__.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 55e4b8db..55ab1f10 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -386,6 +386,8 @@ def _get_package_mirror_info(mirror_info, availability_zone=None, # given a arch specific 'mirror_info' entry (from package_mirrors) # search through the 'search' entries, and fallback appropriately # return a dict with only {name: mirror} entries. + if not mirror_info: + mirror_info = {} ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % "north|northeast|east|southeast|south|southwest|west|northwest") -- cgit v1.2.3 From a7a9de1a079a70f5c8290ab5158661d3a33e5552 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 28 Sep 2012 16:31:50 -0400 Subject: add 'safeyaml' to cloudinit In 0.7.0 we started using yaml.safe_load to load data rather than yaml.load. Some producers (namely, ubuntu MAAS created) have produced cloud-config data in the past that included python unicode types. This creates a specialized safe_loader that is basically safe_load + support for python unicode. --- cloudinit/safeyaml.py | 31 +++++++++++++++++++++++++++++++ cloudinit/util.py | 3 ++- tests/unittests/test_util.py | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 cloudinit/safeyaml.py (limited to 'cloudinit') diff --git a/cloudinit/safeyaml.py b/cloudinit/safeyaml.py new file mode 100644 index 00000000..8b4da1fa --- /dev/null +++ b/cloudinit/safeyaml.py @@ -0,0 +1,31 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import yaml + + +class _CustomSafeLoader(yaml.SafeLoader): + def construct_python_unicode(self, node): + return self.construct_scalar(node) + +_CustomSafeLoader.add_constructor( + u'tag:yaml.org,2002:python/unicode', + _CustomSafeLoader.construct_python_unicode) + +def load(blob): + return(yaml.load(blob, Loader=_CustomSafeLoader)) diff --git a/cloudinit/util.py b/cloudinit/util.py index 94b17dfa..46d490f7 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -50,6 +50,7 @@ import yaml from cloudinit import importer from cloudinit import log as logging +from cloudinit import safeyaml from cloudinit import url_helper as uhelp from cloudinit.settings import (CFG_BUILTIN) @@ -612,7 +613,7 @@ def load_yaml(blob, default=None, allowed=(dict,)): LOG.debug(("Attempting to load yaml from string " "of length %s with allowed root types %s"), len(blob), allowed) - converted = yaml.safe_load(blob) + converted = safeyaml.load(blob) if not isinstance(converted, allowed): # Yes this will just be caught, but thats ok for now... raise TypeError(("Yaml load allows %s root types," diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 15fcbd26..96962b91 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -1,5 +1,6 @@ import os import stat +import yaml from mocker import MockerTestCase from unittest import TestCase @@ -268,4 +269,42 @@ class TestGetCmdline(TestCase): os.environ['DEBUG_PROC_CMDLINE'] = 'abcd 123' self.assertEqual(os.environ['DEBUG_PROC_CMDLINE'], util.get_cmdline()) + +class TestLoadYaml(TestCase): + mydefault = "7b03a8ebace993d806255121073fed52" + + def test_simple(self): + mydata = {'1': "one", '2': "two"} + self.assertEqual(util.load_yaml(yaml.dump(mydata)), mydata) + + def test_nonallowed_returns_default(self): + # for now, anything not in the allowed list just returns the default. + myyaml = yaml.dump({'1': "one"}) + self.assertEqual(util.load_yaml(blob=myyaml, + default=self.mydefault, + allowed=(str,)), + self.mydefault) + + def test_bogus_returns_default(self): + badyaml = "1\n 2:" + self.assertEqual(util.load_yaml(blob=badyaml, + default=self.mydefault), + self.mydefault) + + def test_unsafe_types(self): + # should not load complex types + unsafe_yaml = yaml.dump((1, 2, 3,)) + self.assertEqual(util.load_yaml(blob=unsafe_yaml, + default=self.mydefault), + self.mydefault) + + def test_python_unicode(self): + # complex type of python/unicde is explicitly allowed + myobj = {'1': unicode("FOOBAR")} + safe_yaml = yaml.dump(myobj) + self.assertEqual(util.load_yaml(blob=safe_yaml, + default=self.mydefault), + myobj) + + # vi: ts=4 expandtab -- cgit v1.2.3 From f1e3ae3c49b9424a7e3cdbf835651720cc60e143 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 28 Sep 2012 16:35:53 -0400 Subject: fix pep8 and pylint --- cloudinit/config/cc_ssh_authkey_fingerprints.py | 3 ++- cloudinit/config/cc_users_groups.py | 3 +-- cloudinit/safeyaml.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 2b9a6e0e..f9bdf5fc 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -95,4 +95,5 @@ def handle(name, cfg, cloud, log, _args): (users, _groups) = distros.normalize_users_groups(cfg, cloud.distro) for (user_name, _cfg) in users.items(): (auth_key_fn, auth_key_entries) = extract_func(user_name, cloud.paths) - _pprint_key_entries(user_name, auth_key_fn, auth_key_entries, hash_meth) + _pprint_key_entries(user_name, auth_key_fn, auth_key_entries, + hash_meth) diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index 464f55c3..aa6e0579 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -17,14 +17,13 @@ # along with this program. If not, see . from cloudinit import distros -from cloudinit import util from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -def handle(name, cfg, cloud, log, _args): +def handle(name, cfg, cloud, _log, _args): (users, groups) = distros.normalize_users_groups(cfg, cloud.distro) for (name, members) in groups.items(): cloud.distro.create_group(name, members) diff --git a/cloudinit/safeyaml.py b/cloudinit/safeyaml.py index 8b4da1fa..eba5d056 100644 --- a/cloudinit/safeyaml.py +++ b/cloudinit/safeyaml.py @@ -27,5 +27,6 @@ _CustomSafeLoader.add_constructor( u'tag:yaml.org,2002:python/unicode', _CustomSafeLoader.construct_python_unicode) + def load(blob): return(yaml.load(blob, Loader=_CustomSafeLoader)) -- cgit v1.2.3 From d285a0463b6d16487eb5859373ccfd27eaec8b90 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 28 Sep 2012 16:54:22 -0400 Subject: make DataSourceMAAS 'main()' use load_yaml --- cloudinit/sources/DataSourceMAAS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 581e9a4b..c172150b 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -338,7 +338,7 @@ if __name__ == "__main__": if args.config: import yaml with open(args.config) as fp: - cfg = yaml.safe_load(fp) + cfg = util.load_yaml(fp.read()) if 'datasource' in cfg: cfg = cfg['datasource']['MAAS'] for key in creds.keys(): -- cgit v1.2.3