From 4e36bf2225d2b6e7a7ac1821e1d00ac72abb5856 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 25 Sep 2012 13:19:36 -0700 Subject: Add the ability to pass patches to the rpm builder utility. --- packages/brpm | 7 +++++++ packages/redhat/cloud-init.spec.in | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/packages/brpm b/packages/brpm index 77de0cf2..616ae021 100755 --- a/packages/brpm +++ b/packages/brpm @@ -149,6 +149,7 @@ def generate_spec_contents(args, tmpl_fn, arc_fn): subs['systemd'] = False subs['init_sys'] = args.boot + subs['patches'] = [os.path.basename(p) for p in args.patches] return templater.render_from_file(tmpl_fn, params=subs) @@ -164,6 +165,10 @@ def main(): " (default: %(default)s)"), default=False, action='store_true') + parser.add_argument("-p", "--patch", dest="patches", + help=("include the following patch when building"), + default=[], + action='append') args = parser.parse_args() capture = True if args.verbose: @@ -197,6 +202,8 @@ def main(): spec_fn = util.abs_join(root_dir, 'cloud-init.spec') util.write_file(spec_fn, contents) print("Created spec file at %r" % (spec_fn)) + for p in args.patches: + util.copy(p, util.abs_join(arc_dir, os.path.basename(p)) # Now build it! print("Running 'rpmbuild' in %r" % (root_dir)) diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index 35b27beb..6604b1ac 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -36,6 +36,13 @@ Requires: shadow-utils Requires: ${r} #end for +# Custom patches +#set $size 0 +#for $p in $patches +Patch${size}: $p +#set $size += 1 +#end for + #if $sysvinit Requires(post): chkconfig Requires(postun): initscripts -- cgit v1.2.3 From f74c738fa76db65e45d2bdb19257fe0a78dcd0e0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 25 Sep 2012 13:20:33 -0700 Subject: Fix syntax error. --- packages/brpm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/brpm b/packages/brpm index 616ae021..3870e188 100755 --- a/packages/brpm +++ b/packages/brpm @@ -203,7 +203,7 @@ def main(): util.write_file(spec_fn, contents) print("Created spec file at %r" % (spec_fn)) for p in args.patches: - util.copy(p, util.abs_join(arc_dir, os.path.basename(p)) + util.copy(p, util.abs_join(arc_dir, os.path.basename(p))) # Now build it! print("Running 'rpmbuild' in %r" % (root_dir)) -- cgit v1.2.3 From cdba22dd0fc249124dfb4e578f44898ba044925a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 25 Sep 2012 13:29:07 -0700 Subject: Allow package_mirrors to be non-existent. --- cloudinit/distros/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 549c1612..6df843ad 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -112,7 +112,7 @@ class Distro(object): return arch def _get_arch_package_mirror_info(self, arch=None): - mirror_info = self.get_option("package_mirrors", None) + mirror_info = self.get_option("package_mirrors") or [] if arch == None: arch = self.get_primary_arch() return _get_arch_package_mirror_info(mirror_info, arch) @@ -122,7 +122,6 @@ class Distro(object): # this resolves the package_mirrors config option # down to a single dict of {mirror_name: mirror_url} arch_info = self._get_arch_package_mirror_info(arch) - return _get_package_mirror_info(availability_zone=availability_zone, mirror_info=arch_info) -- cgit v1.2.3 From ca9251d74a18b47abacff2b63328061b9e484246 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 25 Sep 2012 13:37:09 -0700 Subject: Fix cheetah syntax error. --- packages/redhat/cloud-init.spec.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index 6604b1ac..9cfe56f5 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -37,7 +37,7 @@ Requires: ${r} #end for # Custom patches -#set $size 0 +#set $size = 0 #for $p in $patches Patch${size}: $p #set $size += 1 -- cgit v1.2.3 From 018ec6faa0a9d0c8aa584bae2e0798ed7e3e021a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 25 Sep 2012 18:12:26 -0700 Subject: Add a top_dir define. --- packages/brpm | 5 +++-- packages/redhat/cloud-init.spec.in | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/brpm b/packages/brpm index 3870e188..e344926f 100755 --- a/packages/brpm +++ b/packages/brpm @@ -91,7 +91,7 @@ def format_change_line(ds, who, comment=None): return "* %s" % (d) -def generate_spec_contents(args, tmpl_fn, arc_fn): +def generate_spec_contents(args, tmpl_fn, top_dir, arc_fn): # Figure out the version and revno cmd = [util.abs_join(find_root(), 'tools', 'read-version')] @@ -148,6 +148,7 @@ def generate_spec_contents(args, tmpl_fn, arc_fn): else: subs['systemd'] = False + subs['defines'] = ["_topdir %s" % (top_dir)] subs['init_sys'] = args.boot subs['patches'] = [os.path.basename(p) for p in args.patches] return templater.render_from_file(tmpl_fn, params=subs) @@ -197,7 +198,7 @@ def main(): # Form the spec file to be used tmpl_fn = util.abs_join(find_root(), 'packages', 'redhat', 'cloud-init.spec.in') - contents = generate_spec_contents(args, tmpl_fn, + contents = generate_spec_contents(args, tmpl_fn, root_dir, os.path.basename(archive_fn)) spec_fn = util.abs_join(root_dir, 'cloud-init.spec') util.write_file(spec_fn, contents) diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index 9cfe56f5..f5339640 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -5,6 +5,10 @@ # Or: http://fedoraproject.org/wiki/Packaging:ScriptletSnippets # Or: http://www.rpm.org/max-rpm/ch-rpm-inside.html +#for $d in $defines +%define ${d} +#end for + Name: cloud-init Version: ${version} Release: ${release}%{?dist} -- cgit v1.2.3 From c62675900a96dd645a1ff2ee0c316ef9232f443d Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 25 Sep 2012 18:28:12 -0700 Subject: Clean doesn't seem needed. --- packages/brpm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/brpm b/packages/brpm index e344926f..1735f5bb 100755 --- a/packages/brpm +++ b/packages/brpm @@ -208,8 +208,7 @@ def main(): # Now build it! print("Running 'rpmbuild' in %r" % (root_dir)) - cmd = ['rpmbuild', '--clean', - '-ba', spec_fn] + cmd = ['rpmbuild', '-ba', spec_fn] util.subp(cmd, capture=capture) # Copy the items built to our local dir -- cgit v1.2.3 From 179f48250952501c8ad36ac0ed6389159a25bf82 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 26 Sep 2012 16:59:28 -0700 Subject: Ensure the patches get activated. --- packages/redhat/cloud-init.spec.in | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index f5339640..90e85a05 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -69,6 +69,13 @@ ssh keys and to let the user run various scripts. %prep %setup -q -n %{name}-%{version}~${release} +# Custom patches activation +#set $size = 0 +#for $p in $patches +%patch${size} +#set $size += 1 +#end for + %build %{__python} setup.py build -- cgit v1.2.3 From 68114a57d776585865056ad5fb71d41bb741cd06 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 26 Sep 2012 17:03:07 -0700 Subject: Add a nice '-p1' --- packages/redhat/cloud-init.spec.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index 90e85a05..12f0b144 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -72,7 +72,7 @@ ssh keys and to let the user run various scripts. # Custom patches activation #set $size = 0 #for $p in $patches -%patch${size} +%patch${size} -p1 #set $size += 1 #end for -- cgit v1.2.3 From 332ef83d77853714b88358de301bda30e82aaf62 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 28 Sep 2012 11:06:16 -0700 Subject: Stop the bad habit of using 'or'. --- cloudinit/distros/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 6df843ad..55e4b8db 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -112,7 +112,7 @@ class Distro(object): return arch def _get_arch_package_mirror_info(self, arch=None): - mirror_info = self.get_option("package_mirrors") or [] + mirror_info = self.get_option("package_mirrors", []) if arch == None: arch = self.get_primary_arch() return _get_arch_package_mirror_info(mirror_info, arch) -- cgit v1.2.3 From 46be69003044a7d60d9566dbddd1b7fd93054c8f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 28 Sep 2012 11:08:46 -0700 Subject: Also ensure that if mirror info is not invalid before we start iterating over it (it could be sent is as none). --- cloudinit/distros/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 55e4b8db..55ab1f10 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -386,6 +386,8 @@ def _get_package_mirror_info(mirror_info, availability_zone=None, # given a arch specific 'mirror_info' entry (from package_mirrors) # search through the 'search' entries, and fallback appropriately # return a dict with only {name: mirror} entries. + if not mirror_info: + mirror_info = {} ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % "north|northeast|east|southeast|south|southwest|west|northwest") -- cgit v1.2.3 From a7a9de1a079a70f5c8290ab5158661d3a33e5552 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 28 Sep 2012 16:31:50 -0400 Subject: add 'safeyaml' to cloudinit In 0.7.0 we started using yaml.safe_load to load data rather than yaml.load. Some producers (namely, ubuntu MAAS created) have produced cloud-config data in the past that included python unicode types. This creates a specialized safe_loader that is basically safe_load + support for python unicode. --- cloudinit/safeyaml.py | 31 +++++++++++++++++++++++++++++++ cloudinit/util.py | 3 ++- tests/unittests/test_util.py | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 cloudinit/safeyaml.py diff --git a/cloudinit/safeyaml.py b/cloudinit/safeyaml.py new file mode 100644 index 00000000..8b4da1fa --- /dev/null +++ b/cloudinit/safeyaml.py @@ -0,0 +1,31 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import yaml + + +class _CustomSafeLoader(yaml.SafeLoader): + def construct_python_unicode(self, node): + return self.construct_scalar(node) + +_CustomSafeLoader.add_constructor( + u'tag:yaml.org,2002:python/unicode', + _CustomSafeLoader.construct_python_unicode) + +def load(blob): + return(yaml.load(blob, Loader=_CustomSafeLoader)) diff --git a/cloudinit/util.py b/cloudinit/util.py index 94b17dfa..46d490f7 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -50,6 +50,7 @@ import yaml from cloudinit import importer from cloudinit import log as logging +from cloudinit import safeyaml from cloudinit import url_helper as uhelp from cloudinit.settings import (CFG_BUILTIN) @@ -612,7 +613,7 @@ def load_yaml(blob, default=None, allowed=(dict,)): LOG.debug(("Attempting to load yaml from string " "of length %s with allowed root types %s"), len(blob), allowed) - converted = yaml.safe_load(blob) + converted = safeyaml.load(blob) if not isinstance(converted, allowed): # Yes this will just be caught, but thats ok for now... raise TypeError(("Yaml load allows %s root types," diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 15fcbd26..96962b91 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -1,5 +1,6 @@ import os import stat +import yaml from mocker import MockerTestCase from unittest import TestCase @@ -268,4 +269,42 @@ class TestGetCmdline(TestCase): os.environ['DEBUG_PROC_CMDLINE'] = 'abcd 123' self.assertEqual(os.environ['DEBUG_PROC_CMDLINE'], util.get_cmdline()) + +class TestLoadYaml(TestCase): + mydefault = "7b03a8ebace993d806255121073fed52" + + def test_simple(self): + mydata = {'1': "one", '2': "two"} + self.assertEqual(util.load_yaml(yaml.dump(mydata)), mydata) + + def test_nonallowed_returns_default(self): + # for now, anything not in the allowed list just returns the default. + myyaml = yaml.dump({'1': "one"}) + self.assertEqual(util.load_yaml(blob=myyaml, + default=self.mydefault, + allowed=(str,)), + self.mydefault) + + def test_bogus_returns_default(self): + badyaml = "1\n 2:" + self.assertEqual(util.load_yaml(blob=badyaml, + default=self.mydefault), + self.mydefault) + + def test_unsafe_types(self): + # should not load complex types + unsafe_yaml = yaml.dump((1, 2, 3,)) + self.assertEqual(util.load_yaml(blob=unsafe_yaml, + default=self.mydefault), + self.mydefault) + + def test_python_unicode(self): + # complex type of python/unicde is explicitly allowed + myobj = {'1': unicode("FOOBAR")} + safe_yaml = yaml.dump(myobj) + self.assertEqual(util.load_yaml(blob=safe_yaml, + default=self.mydefault), + myobj) + + # vi: ts=4 expandtab -- cgit v1.2.3 From f1e3ae3c49b9424a7e3cdbf835651720cc60e143 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 28 Sep 2012 16:35:53 -0400 Subject: fix pep8 and pylint --- cloudinit/config/cc_ssh_authkey_fingerprints.py | 3 ++- cloudinit/config/cc_users_groups.py | 3 +-- cloudinit/safeyaml.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 2b9a6e0e..f9bdf5fc 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -95,4 +95,5 @@ def handle(name, cfg, cloud, log, _args): (users, _groups) = distros.normalize_users_groups(cfg, cloud.distro) for (user_name, _cfg) in users.items(): (auth_key_fn, auth_key_entries) = extract_func(user_name, cloud.paths) - _pprint_key_entries(user_name, auth_key_fn, auth_key_entries, hash_meth) + _pprint_key_entries(user_name, auth_key_fn, auth_key_entries, + hash_meth) diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index 464f55c3..aa6e0579 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -17,14 +17,13 @@ # along with this program. If not, see . from cloudinit import distros -from cloudinit import util from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -def handle(name, cfg, cloud, log, _args): +def handle(name, cfg, cloud, _log, _args): (users, groups) = distros.normalize_users_groups(cfg, cloud.distro) for (name, members) in groups.items(): cloud.distro.create_group(name, members) diff --git a/cloudinit/safeyaml.py b/cloudinit/safeyaml.py index 8b4da1fa..eba5d056 100644 --- a/cloudinit/safeyaml.py +++ b/cloudinit/safeyaml.py @@ -27,5 +27,6 @@ _CustomSafeLoader.add_constructor( u'tag:yaml.org,2002:python/unicode', _CustomSafeLoader.construct_python_unicode) + def load(blob): return(yaml.load(blob, Loader=_CustomSafeLoader)) -- cgit v1.2.3 From d285a0463b6d16487eb5859373ccfd27eaec8b90 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 28 Sep 2012 16:54:22 -0400 Subject: make DataSourceMAAS 'main()' use load_yaml --- cloudinit/sources/DataSourceMAAS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 581e9a4b..c172150b 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -338,7 +338,7 @@ if __name__ == "__main__": if args.config: import yaml with open(args.config) as fp: - cfg = yaml.safe_load(fp) + cfg = util.load_yaml(fp.read()) if 'datasource' in cfg: cfg = cfg['datasource']['MAAS'] for key in creds.keys(): -- cgit v1.2.3