From e14d64a03c6aa3e567b57f0c0a003ca2185f4493 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 23 Apr 2013 22:53:53 -0700 Subject: Add a bunch of new merging test files + runner. --- tests/data/merge_sources/expected1.yaml | 1 + tests/data/merge_sources/expected2.yaml | 3 +++ tests/data/merge_sources/expected3.yaml | 1 + tests/data/merge_sources/expected4.yaml | 2 ++ tests/data/merge_sources/source1-1.yaml | 3 +++ tests/data/merge_sources/source1-2.yaml | 5 +++++ tests/data/merge_sources/source2-1.yaml | 6 ++++++ tests/data/merge_sources/source2-2.yaml | 5 +++++ tests/data/merge_sources/source3-1.yaml | 4 ++++ tests/data/merge_sources/source3-2.yaml | 4 ++++ tests/data/merge_sources/source4-1.yaml | 3 +++ tests/data/merge_sources/source4-2.yaml | 6 ++++++ 12 files changed, 43 insertions(+) create mode 100644 tests/data/merge_sources/expected1.yaml create mode 100644 tests/data/merge_sources/expected2.yaml create mode 100644 tests/data/merge_sources/expected3.yaml create mode 100644 tests/data/merge_sources/expected4.yaml create mode 100644 tests/data/merge_sources/source1-1.yaml create mode 100644 tests/data/merge_sources/source1-2.yaml create mode 100644 tests/data/merge_sources/source2-1.yaml create mode 100644 tests/data/merge_sources/source2-2.yaml create mode 100644 tests/data/merge_sources/source3-1.yaml create mode 100644 tests/data/merge_sources/source3-2.yaml create mode 100644 tests/data/merge_sources/source4-1.yaml create mode 100644 tests/data/merge_sources/source4-2.yaml (limited to 'tests/data') diff --git a/tests/data/merge_sources/expected1.yaml b/tests/data/merge_sources/expected1.yaml new file mode 100644 index 00000000..640d282b --- /dev/null +++ b/tests/data/merge_sources/expected1.yaml @@ -0,0 +1 @@ +Blah: ['blah2', 'b'] diff --git a/tests/data/merge_sources/expected2.yaml b/tests/data/merge_sources/expected2.yaml new file mode 100644 index 00000000..6eccc2cf --- /dev/null +++ b/tests/data/merge_sources/expected2.yaml @@ -0,0 +1,3 @@ +Blah: 3 +Blah2: 2 +Blah3: [1] diff --git a/tests/data/merge_sources/expected3.yaml b/tests/data/merge_sources/expected3.yaml new file mode 100644 index 00000000..32d9ad48 --- /dev/null +++ b/tests/data/merge_sources/expected3.yaml @@ -0,0 +1 @@ +Blah: [blah2, 'blah1'] diff --git a/tests/data/merge_sources/expected4.yaml b/tests/data/merge_sources/expected4.yaml new file mode 100644 index 00000000..d88d8f73 --- /dev/null +++ b/tests/data/merge_sources/expected4.yaml @@ -0,0 +1,2 @@ +#cloud-config +Blah: {} diff --git a/tests/data/merge_sources/source1-1.yaml b/tests/data/merge_sources/source1-1.yaml new file mode 100644 index 00000000..38e4e5e0 --- /dev/null +++ b/tests/data/merge_sources/source1-1.yaml @@ -0,0 +1,3 @@ +#cloud-config +Blah: ['blah2'] + diff --git a/tests/data/merge_sources/source1-2.yaml b/tests/data/merge_sources/source1-2.yaml new file mode 100644 index 00000000..2cd0e0e5 --- /dev/null +++ b/tests/data/merge_sources/source1-2.yaml @@ -0,0 +1,5 @@ +#cloud-config + +Blah: ['b'] + +merge_how: 'dict(recurse_array,no_replace)+list(append)' diff --git a/tests/data/merge_sources/source2-1.yaml b/tests/data/merge_sources/source2-1.yaml new file mode 100644 index 00000000..c7a33aaa --- /dev/null +++ b/tests/data/merge_sources/source2-1.yaml @@ -0,0 +1,6 @@ +#cloud-config + + +Blah: 1 +Blah2: 2 +Blah3: 3 diff --git a/tests/data/merge_sources/source2-2.yaml b/tests/data/merge_sources/source2-2.yaml new file mode 100644 index 00000000..8f2fdc1a --- /dev/null +++ b/tests/data/merge_sources/source2-2.yaml @@ -0,0 +1,5 @@ +#cloud-config + +Blah: 3 +Blah2: 2 +Blah3: [1] diff --git a/tests/data/merge_sources/source3-1.yaml b/tests/data/merge_sources/source3-1.yaml new file mode 100644 index 00000000..2303e906 --- /dev/null +++ b/tests/data/merge_sources/source3-1.yaml @@ -0,0 +1,4 @@ +#cloud-config +Blah: ['blah1'] + + diff --git a/tests/data/merge_sources/source3-2.yaml b/tests/data/merge_sources/source3-2.yaml new file mode 100644 index 00000000..dca2ad10 --- /dev/null +++ b/tests/data/merge_sources/source3-2.yaml @@ -0,0 +1,4 @@ +#cloud-config +Blah: ['blah2'] + +merge_how: 'dict(recurse_array,no_replace)+list(prepend)' diff --git a/tests/data/merge_sources/source4-1.yaml b/tests/data/merge_sources/source4-1.yaml new file mode 100644 index 00000000..e5b16872 --- /dev/null +++ b/tests/data/merge_sources/source4-1.yaml @@ -0,0 +1,3 @@ +#cloud-config +Blah: + b: 1 diff --git a/tests/data/merge_sources/source4-2.yaml b/tests/data/merge_sources/source4-2.yaml new file mode 100644 index 00000000..1844e0f8 --- /dev/null +++ b/tests/data/merge_sources/source4-2.yaml @@ -0,0 +1,6 @@ +#cloud-config +Blah: + b: null + + +merge_how: 'dict(allow_delete,no_replace)+list()' -- cgit v1.2.3 From 4a669649b17cf01b6f89f7902b6683d02ef0bee1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 3 May 2013 14:41:28 -0700 Subject: More merging adjustments. Looks like this should be in pretty good shape and has passed some of the basic backwards compat. merging tests that I added. --- cloudinit/mergers/__init__.py | 2 +- cloudinit/mergers/m_dict.py | 27 ++++--- cloudinit/mergers/m_list.py | 9 +-- cloudinit/mergers/m_str.py | 44 ++++++++++++ tests/data/merge_sources/expected2.yaml | 4 +- tests/data/merge_sources/expected5.yaml | 7 ++ tests/data/merge_sources/source5-1.yaml | 6 ++ tests/data/merge_sources/source5-2.yaml | 8 +++ tests/unittests/test_merging.py | 123 +++++++++++++++++++++++++++----- tests/unittests/test_userdata.py | 5 +- 10 files changed, 201 insertions(+), 34 deletions(-) create mode 100644 cloudinit/mergers/m_str.py create mode 100644 tests/data/merge_sources/expected5.yaml create mode 100644 tests/data/merge_sources/source5-1.yaml create mode 100644 tests/data/merge_sources/source5-2.yaml (limited to 'tests/data') diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index 221e93b5..0978b2c6 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -25,7 +25,7 @@ from cloudinit import type_utils NAME_MTCH = re.compile(r"(^[a-zA-Z_][A-Za-z0-9_]*)\((.*?)\)$") LOG = logging.getLogger(__name__) -DEF_MERGE_TYPE = "list()+dict()" +DEF_MERGE_TYPE = "list()+dict()+str()" MERGER_PREFIX = 'm_' MERGER_ATTR = 'Merger' diff --git a/cloudinit/mergers/m_dict.py b/cloudinit/mergers/m_dict.py index 2c1c845f..82caa004 100644 --- a/cloudinit/mergers/m_dict.py +++ b/cloudinit/mergers/m_dict.py @@ -16,21 +16,32 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +DEF_MERGE_TYPE = 'no_replace' +MERGE_TYPES = ('replace', DEF_MERGE_TYPE,) + + +def _has_any(what, *keys): + for k in keys: + if k in what: + return True + return False + class Merger(object): def __init__(self, merger, opts): self._merger = merger # Affects merging behavior... - self._method = 'replace' - for m in ['replace', 'no_replace']: + self._method = DEF_MERGE_TYPE + for m in MERGE_TYPES: if m in opts: self._method = m break - # Affect how recursive merging is done on other primitives + # Affect how recursive merging is done on other primitives. self._recurse_str = 'recurse_str' in opts - self._recurse_dict = True - self._recurse_array = 'recurse_array' in opts + self._recurse_array = _has_any(opts, 'recurse_array', 'recurse_list') self._allow_delete = 'allow_delete' in opts + # Backwards compat require this to be on. + self._recurse_dict = True def __str__(self): s = ('DictMerger: (method=%s,recurse_str=%s,' @@ -42,14 +53,14 @@ class Merger(object): self._allow_delete) return s - def _do_dict_replace(self, value, merge_with, do_replace=True): + def _do_dict_replace(self, value, merge_with, do_replace): def merge_same_key(old_v, new_v): if do_replace: return new_v if isinstance(new_v, (list, tuple)) and self._recurse_array: return self._merger.merge(old_v, new_v) - if isinstance(new_v, (str, basestring)) and self._recurse_str: + if isinstance(new_v, (basestring)) and self._recurse_str: return self._merger.merge(old_v, new_v) if isinstance(new_v, (dict)) and self._recurse_dict: return self._merger.merge(old_v, new_v) @@ -70,7 +81,7 @@ class Merger(object): if not isinstance(merge_with, (dict)): return value if self._method == 'replace': - merged = self._do_dict_replace(dict(value), merge_with) + merged = self._do_dict_replace(dict(value), merge_with, True) elif self._method == 'no_replace': merged = self._do_dict_replace(dict(value), merge_with, False) else: diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index c6a23d85..50f279e8 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -32,10 +32,11 @@ class Merger(object): self._recurse_array = 'recurse_array' in opts def __str__(self): - return 'ListMerger: (m=%s,rs=%s,rd=%s,ra=%s)' % (self._method, - self._recurse_str, - self._recurse_dict, - self._recurse_array) + return ('ListMerger: (method=%s,recurse_str=%s,' + 'recurse_dict=%s,recurse_array=%s)') % (self._method, + self._recurse_str, + self._recurse_dict, + self._recurse_array) def _on_tuple(self, value, merge_with): return tuple(self._on_list(list(value), merge_with)) diff --git a/cloudinit/mergers/m_str.py b/cloudinit/mergers/m_str.py new file mode 100644 index 00000000..e22ce28a --- /dev/null +++ b/cloudinit/mergers/m_str.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class Merger(object): + def __init__(self, _merger, opts): + self._append = 'append' in opts + + def __str__(self): + return 'StringMerger: (append=%s)' % (self._append) + + # On encountering a unicode object to merge value with + # we will for now just proxy into the string method to let it handle it. + def _on_unicode(self, value, merge_with): + return self._on_str(value, merge_with) + + # On encountering a string object to merge with we will + # perform the following action, if appending we will + # merge them together, otherwise we will just return value. + def _on_str(self, value, merge_with): + if not isinstance(value, (basestring)): + return merge_with + if not self._append: + return merge_with + if isinstance(value, unicode): + return value + unicode(merge_with) + else: + return value + str(merge_with) diff --git a/tests/data/merge_sources/expected2.yaml b/tests/data/merge_sources/expected2.yaml index 6eccc2cf..f5312eb1 100644 --- a/tests/data/merge_sources/expected2.yaml +++ b/tests/data/merge_sources/expected2.yaml @@ -1,3 +1,3 @@ -Blah: 3 +Blah: 1 Blah2: 2 -Blah3: [1] +Blah3: 3 diff --git a/tests/data/merge_sources/expected5.yaml b/tests/data/merge_sources/expected5.yaml new file mode 100644 index 00000000..628f5878 --- /dev/null +++ b/tests/data/merge_sources/expected5.yaml @@ -0,0 +1,7 @@ +#cloud-config + +Blah: 3 +Blah2: 2 +Blah3: [1] + + diff --git a/tests/data/merge_sources/source5-1.yaml b/tests/data/merge_sources/source5-1.yaml new file mode 100644 index 00000000..c7a33aaa --- /dev/null +++ b/tests/data/merge_sources/source5-1.yaml @@ -0,0 +1,6 @@ +#cloud-config + + +Blah: 1 +Blah2: 2 +Blah3: 3 diff --git a/tests/data/merge_sources/source5-2.yaml b/tests/data/merge_sources/source5-2.yaml new file mode 100644 index 00000000..f61c96a2 --- /dev/null +++ b/tests/data/merge_sources/source5-2.yaml @@ -0,0 +1,8 @@ +#cloud-config + +Blah: 3 +Blah2: 2 +Blah3: [1] + + +merge_how: 'dict(replace)+list(append)' diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 470b18c7..f83522d7 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -11,14 +11,39 @@ import glob import os import re +SOURCE_PAT = "source*.*yaml" +EXPECTED_PAT = "expected%s.yaml" + + +def _old_mergedict(src, cand): + """ + Merge values from C{cand} into C{src}. + If C{src} has a key C{cand} will not override. + Nested dictionaries are merged recursively. + """ + if isinstance(src, dict) and isinstance(cand, dict): + for (k, v) in cand.iteritems(): + if k not in src: + src[k] = v + else: + src[k] = _old_mergedict(src[k], v) + return src + + +def _old_mergemanydict(*args): + out = {} + for a in args: + out = _old_mergedict(out, a) + return out + class TestSimpleRun(helpers.ResourceUsingTestCase): - def _load_merge_files(self, data_dir): - merge_root = self.resourceLocation(data_dir) + def _load_merge_files(self): + merge_root = self.resourceLocation('merge_sources') tests = [] source_ids = collections.defaultdict(list) expected_files = {} - for fn in glob.glob(os.path.join(merge_root, "source*.*yaml")): + for fn in glob.glob(os.path.join(merge_root, SOURCE_PAT)): base_fn = os.path.basename(fn) file_id = re.match(r"source(\d+)\-(\d+)[.]yaml", base_fn) if not file_id: @@ -26,31 +51,97 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): % (fn)) file_id = int(file_id.group(1)) source_ids[file_id].append(fn) - expected_fn = os.path.join(merge_root, - "expected%s.yaml" % (file_id)) + expected_fn = os.path.join(merge_root, EXPECTED_PAT % (file_id)) if not os.path.isfile(expected_fn): raise IOError("No expected file found at %s" % (expected_fn)) expected_files[file_id] = expected_fn - for id in sorted(source_ids.keys()): + for i in sorted(source_ids.keys()): source_file_contents = [] - for fn in sorted(source_ids[id]): - source_file_contents.append(util.load_file(fn)) - expected = util.load_yaml(util.load_file(expected_files[id])) - tests.append((source_file_contents, expected)) + for fn in sorted(source_ids[i]): + source_file_contents.append([fn, util.load_file(fn)]) + expected = util.load_yaml(util.load_file(expected_files[i])) + entry = [source_file_contents, [expected, expected_files[i]]] + tests.append(entry) return tests def test_merge_samples(self): - tests = self._load_merge_files('merge_sources') + tests = self._load_merge_files() paths = c_helpers.Paths({}) cc_handler = cloud_config.CloudConfigPartHandler(paths) cc_handler.cloud_fn = None - for (payloads, expected_merge) in tests: + for (payloads, (expected_merge, expected_fn)) in tests: cc_handler.handle_part(None, CONTENT_START, None, None, None, None) - for (i, p) in enumerate(payloads): - cc_handler.handle_part(None, None, "t-%s.yaml" % (i + 1), - p, None, {}) + merging_fns = [] + for (fn, contents) in payloads: + cc_handler.handle_part(None, None, "%s.yaml" % (fn), + contents, None, {}) + merging_fns.append(fn) merged_buf = cc_handler.cloud_buf cc_handler.handle_part(None, CONTENT_END, None, None, None, None) - self.assertEquals(expected_merge, merged_buf) + fail_msg = "Equality failure on checking %s with %s: %s != %s" + fail_msg = fail_msg % (expected_fn, + ",".join(merging_fns), merged_buf, + expected_merge) + self.assertEquals(expected_merge, merged_buf, msg=fail_msg) + + def test_compat_merges_dict(self): + a = { + '1': '2', + 'b': 'c', + } + b = { + 'b': 'e', + } + c = _old_mergedict(a, b) + d = util.mergemanydict([a, b]) + self.assertEquals(c, d) + + def test_compat_merges_list(self): + a = {'b': [1, 2, 3]} + b = {'b': [4, 5]} + c = {'b': [6, 7]} + e = _old_mergemanydict(a, b, c) + f = util.mergemanydict([a, b, c]) + self.assertEquals(e, f) + + def test_compat_merges_str(self): + a = {'b': "hi"} + b = {'b': "howdy"} + c = {'b': "hallo"} + e = _old_mergemanydict(a, b, c) + f = util.mergemanydict([a, b, c]) + self.assertEquals(e, f) + + def test_compat_merge_sub_dict(self): + a = { + '1': '2', + 'b': { + 'f': 'g', + } + } + b = { + 'b': { + 'e': 'c', + } + } + c = _old_mergedict(a, b) + d = util.mergemanydict([a, b]) + self.assertEquals(c, d) + + def test_compat_merge_sub_list(self): + a = { + '1': '2', + 'b': { + 'f': ['1'], + } + } + b = { + 'b': { + 'f': [], + } + } + c = _old_mergedict(a, b) + d = util.mergemanydict([a, b]) + self.assertEquals(c, d) diff --git a/tests/unittests/test_userdata.py b/tests/unittests/test_userdata.py index fdfe2542..5fb9acd9 100644 --- a/tests/unittests/test_userdata.py +++ b/tests/unittests/test_userdata.py @@ -60,7 +60,6 @@ run: - c ''' message1 = MIMEBase("text", "cloud-config") - message1['Merge-Type'] = 'dict()+list(extend)+str(append)' message1.set_payload(blob) blob2 = ''' @@ -72,7 +71,8 @@ run: - morestuff ''' message2 = MIMEBase("text", "cloud-config") - message2['X-Merge-Type'] = 'dict()+list(extend)+str()' + message2['X-Merge-Type'] = ('dict(recurse_array,' + 'recurse_str)+list(append)+str(append)') message2.set_payload(blob2) blob3 = ''' @@ -84,7 +84,6 @@ e: p: 1 ''' message3 = MIMEBase("text", "cloud-config") - message3['Merge-Type'] = 'dict()+list()+str()' message3.set_payload(blob3) messages = [message1, message2, message3] -- cgit v1.2.3 From 229df67191e7dff058151f1e1f6e007667d55d9c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 7 May 2013 21:38:06 -0700 Subject: A few more test files. --- cloudinit/mergers/m_list.py | 12 ++++++++--- tests/data/merge_sources/expected6.yaml | 9 ++++++++ tests/data/merge_sources/expected7.yaml | 38 +++++++++++++++++++++++++++++++++ tests/data/merge_sources/expected8.yaml | 7 ++++++ tests/data/merge_sources/expected9.yaml | 5 +++++ tests/data/merge_sources/source6-1.yaml | 5 +++++ tests/data/merge_sources/source6-2.yaml | 8 +++++++ tests/data/merge_sources/source7-1.yaml | 27 +++++++++++++++++++++++ tests/data/merge_sources/source7-2.yaml | 17 +++++++++++++++ tests/data/merge_sources/source8-1.yaml | 7 ++++++ tests/data/merge_sources/source8-2.yaml | 6 ++++++ tests/data/merge_sources/source9-1.yaml | 5 +++++ tests/data/merge_sources/source9-2.yaml | 6 ++++++ 13 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 tests/data/merge_sources/expected6.yaml create mode 100644 tests/data/merge_sources/expected7.yaml create mode 100644 tests/data/merge_sources/expected8.yaml create mode 100644 tests/data/merge_sources/expected9.yaml create mode 100644 tests/data/merge_sources/source6-1.yaml create mode 100644 tests/data/merge_sources/source6-2.yaml create mode 100644 tests/data/merge_sources/source7-1.yaml create mode 100644 tests/data/merge_sources/source7-2.yaml create mode 100644 tests/data/merge_sources/source8-1.yaml create mode 100644 tests/data/merge_sources/source8-2.yaml create mode 100644 tests/data/merge_sources/source9-1.yaml create mode 100644 tests/data/merge_sources/source9-2.yaml (limited to 'tests/data') diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index 8a0b5827..1184ded7 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -19,6 +19,12 @@ DEF_MERGE_TYPE = 'replace' MERGE_TYPES = ('append', 'prepend', DEF_MERGE_TYPE,) +def _has_any(what, *keys): + for k in keys: + if k in what: + return True + return False + class Merger(object): def __init__(self, merger, opts): @@ -30,9 +36,9 @@ class Merger(object): self._method = m break # Affect how recursive merging is done on other primitives - self._recurse_str = 'recurse_str' in opts - self._recurse_dict = 'recurse_dict' in opts - self._recurse_array = 'recurse_array' in opts + self._recurse_str = _has_any(opts, 'recurse_str') + self._recurse_dict = _has_any(opts, 'recurse_dict') + self._recurse_array = _has_any(opts, 'recurse_array', 'recurse_list') def __str__(self): return ('ListMerger: (method=%s,recurse_str=%s,' diff --git a/tests/data/merge_sources/expected6.yaml b/tests/data/merge_sources/expected6.yaml new file mode 100644 index 00000000..7afe1d7c --- /dev/null +++ b/tests/data/merge_sources/expected6.yaml @@ -0,0 +1,9 @@ +#cloud-config + +run_cmds: + - bash + - top + - ps + - vi + - emacs + diff --git a/tests/data/merge_sources/expected7.yaml b/tests/data/merge_sources/expected7.yaml new file mode 100644 index 00000000..25284f04 --- /dev/null +++ b/tests/data/merge_sources/expected7.yaml @@ -0,0 +1,38 @@ +#cloud-config + +users: + - default + - name: foobar + gecos: Foo B. Bar + primary-group: foobar + groups: users + selinux-user: staff_u + expiredate: 2012-09-01 + ssh-import-id: foobar + lock-passwd: false + passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ + - name: barfoo + gecos: Bar B. Foo + sudo: ALL=(ALL) NOPASSWD:ALL + groups: users, admin + ssh-import-id: None + lock-passwd: true + ssh-authorized-keys: + - + - + - name: cloudy + gecos: Magic Cloud App Daemon User + inactive: true + system: true + - bob + - joe + - sue + - name: foobar_jr + gecos: Foo B. Bar Jr + primary-group: foobar + groups: users + selinux-user: staff_u + expiredate: 2012-09-01 + ssh-import-id: foobar + lock-passwd: false + passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ diff --git a/tests/data/merge_sources/expected8.yaml b/tests/data/merge_sources/expected8.yaml new file mode 100644 index 00000000..69ca562d --- /dev/null +++ b/tests/data/merge_sources/expected8.yaml @@ -0,0 +1,7 @@ +#cloud-config + +mounts: + - [ ephemeral22, /mnt, auto, "defaults,noexec" ] + - [ sdc, /opt/data ] + - [ xvdh, /opt/data, "auto", "defaults,nobootwait", "0", "0" ] + - [ dd, /dev/zero ] diff --git a/tests/data/merge_sources/expected9.yaml b/tests/data/merge_sources/expected9.yaml new file mode 100644 index 00000000..00f91ca0 --- /dev/null +++ b/tests/data/merge_sources/expected9.yaml @@ -0,0 +1,5 @@ +#cloud-config + +phone_home: + url: http://my.example.com/$INSTANCE_ID/$BLAH_BLAH + post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id ] diff --git a/tests/data/merge_sources/source6-1.yaml b/tests/data/merge_sources/source6-1.yaml new file mode 100644 index 00000000..519f7309 --- /dev/null +++ b/tests/data/merge_sources/source6-1.yaml @@ -0,0 +1,5 @@ +#cloud-config + +run_cmds: + - bash + - top diff --git a/tests/data/merge_sources/source6-2.yaml b/tests/data/merge_sources/source6-2.yaml new file mode 100644 index 00000000..d8fac446 --- /dev/null +++ b/tests/data/merge_sources/source6-2.yaml @@ -0,0 +1,8 @@ +#cloud-config + +run_cmds: + - ps + - vi + - emacs + +merge_type: 'list(append)+dict(recurse_array)+str()' diff --git a/tests/data/merge_sources/source7-1.yaml b/tests/data/merge_sources/source7-1.yaml new file mode 100644 index 00000000..8fb9b32a --- /dev/null +++ b/tests/data/merge_sources/source7-1.yaml @@ -0,0 +1,27 @@ +#cloud-config + +users: + - default + - name: foobar + gecos: Foo B. Bar + primary-group: foobar + groups: users + selinux-user: staff_u + expiredate: 2012-09-01 + ssh-import-id: foobar + lock-passwd: false + passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ + - name: barfoo + gecos: Bar B. Foo + sudo: ALL=(ALL) NOPASSWD:ALL + groups: users, admin + ssh-import-id: None + lock-passwd: true + ssh-authorized-keys: + - + - + - name: cloudy + gecos: Magic Cloud App Daemon User + inactive: true + system: true + diff --git a/tests/data/merge_sources/source7-2.yaml b/tests/data/merge_sources/source7-2.yaml new file mode 100644 index 00000000..1e26201b --- /dev/null +++ b/tests/data/merge_sources/source7-2.yaml @@ -0,0 +1,17 @@ +#cloud-config + +users: + - bob + - joe + - sue + - name: foobar_jr + gecos: Foo B. Bar Jr + primary-group: foobar + groups: users + selinux-user: staff_u + expiredate: 2012-09-01 + ssh-import-id: foobar + lock-passwd: false + passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ + +merge_how: "dict(recurse_array)+list(append)" diff --git a/tests/data/merge_sources/source8-1.yaml b/tests/data/merge_sources/source8-1.yaml new file mode 100644 index 00000000..5ea51c2c --- /dev/null +++ b/tests/data/merge_sources/source8-1.yaml @@ -0,0 +1,7 @@ +#cloud-config + +mounts: + - [ ephemeral0, /mnt, auto, "defaults,noexec" ] + - [ sdc, /opt/data ] + - [ xvdh, /opt/data, "auto", "defaults,nobootwait", "0", "0" ] + - [ dd, /dev/zero ] diff --git a/tests/data/merge_sources/source8-2.yaml b/tests/data/merge_sources/source8-2.yaml new file mode 100644 index 00000000..7fa3262b --- /dev/null +++ b/tests/data/merge_sources/source8-2.yaml @@ -0,0 +1,6 @@ +#cloud-config + +mounts: + - [ ephemeral22, /mnt, auto, "defaults,noexec" ] + +merge_how: 'dict(recurse_array)+list(recurse_list,recurse_str)+str()' diff --git a/tests/data/merge_sources/source9-1.yaml b/tests/data/merge_sources/source9-1.yaml new file mode 100644 index 00000000..0b102ba6 --- /dev/null +++ b/tests/data/merge_sources/source9-1.yaml @@ -0,0 +1,5 @@ +#cloud-config + +phone_home: + url: http://my.example.com/$INSTANCE_ID/ + post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id ] diff --git a/tests/data/merge_sources/source9-2.yaml b/tests/data/merge_sources/source9-2.yaml new file mode 100644 index 00000000..ac85afc6 --- /dev/null +++ b/tests/data/merge_sources/source9-2.yaml @@ -0,0 +1,6 @@ +#cloud-config + +phone_home: + url: $BLAH_BLAH + +merge_how: 'dict(recurse_str)+str(append)' -- cgit v1.2.3 From ff232886555964220769da6d8b73198b5d51ef16 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 7 May 2013 21:42:32 -0700 Subject: 1 more test that does some list appending. --- tests/data/merge_sources/expected10.yaml | 7 +++++++ tests/data/merge_sources/source10-1.yaml | 6 ++++++ tests/data/merge_sources/source10-2.yaml | 6 ++++++ 3 files changed, 19 insertions(+) create mode 100644 tests/data/merge_sources/expected10.yaml create mode 100644 tests/data/merge_sources/source10-1.yaml create mode 100644 tests/data/merge_sources/source10-2.yaml (limited to 'tests/data') diff --git a/tests/data/merge_sources/expected10.yaml b/tests/data/merge_sources/expected10.yaml new file mode 100644 index 00000000..b865db16 --- /dev/null +++ b/tests/data/merge_sources/expected10.yaml @@ -0,0 +1,7 @@ +#cloud-config + +power_state: + delay: 30 + mode: poweroff + message: [Bye, Bye, Pew, Pew] + diff --git a/tests/data/merge_sources/source10-1.yaml b/tests/data/merge_sources/source10-1.yaml new file mode 100644 index 00000000..6ae72a13 --- /dev/null +++ b/tests/data/merge_sources/source10-1.yaml @@ -0,0 +1,6 @@ +#cloud-config + +power_state: + delay: 30 + mode: poweroff + message: [Bye, Bye] diff --git a/tests/data/merge_sources/source10-2.yaml b/tests/data/merge_sources/source10-2.yaml new file mode 100644 index 00000000..a38cf1c5 --- /dev/null +++ b/tests/data/merge_sources/source10-2.yaml @@ -0,0 +1,6 @@ +#cloud-config + +power_state: + message: [Pew, Pew] + +merge_how: 'dict(recurse_list)+list(append)' -- cgit v1.2.3 From 2b351c5435939d16ba06ec0c45847d47f4b21d51 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 9 May 2013 22:34:31 -0700 Subject: Fix the cloud config merging so that it is backwards compat. The new change for merging works well in the mergedict case but the default merging type for cloud config needs to reflect how yaml was loaded in bulk, which is the same as the replacing keys merging type that is now provided. --- cloudinit/handlers/cloud_config.py | 15 +++++++++++++-- tests/data/merge_sources/expected11.yaml | 5 +++++ tests/data/merge_sources/expected12.yaml | 5 +++++ tests/data/merge_sources/expected2.yaml | 4 ++-- tests/data/merge_sources/source11-1.yaml | 5 +++++ tests/data/merge_sources/source11-2.yaml | 3 +++ tests/data/merge_sources/source11-3.yaml | 3 +++ tests/data/merge_sources/source12-1.yaml | 8 ++++++++ tests/data/merge_sources/source12-2.yaml | 5 +++++ tests/unittests/test_merging.py | 15 +++++++++++++++ tests/unittests/test_userdata.py | 2 +- 11 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 tests/data/merge_sources/expected11.yaml create mode 100644 tests/data/merge_sources/expected12.yaml create mode 100644 tests/data/merge_sources/source11-1.yaml create mode 100644 tests/data/merge_sources/source11-2.yaml create mode 100644 tests/data/merge_sources/source11-3.yaml create mode 100644 tests/data/merge_sources/source12-1.yaml create mode 100644 tests/data/merge_sources/source12-2.yaml (limited to 'tests/data') diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 2ae9b226..529109ce 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -30,7 +30,13 @@ from cloudinit.settings import (PER_ALWAYS) LOG = logging.getLogger(__name__) MERGE_HEADER = 'Merge-Type' -DEF_MERGERS = mergers.default_mergers() + +# Due to the way the loading of yaml configuration was done previously, +# where previously each cloud config part was appended to a larger yaml +# file and then finally that file was loaded as one big yaml file we need +# to mimic that behavior by altering the default strategy to be replacing +# keys of later mergers. +DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()') class CloudConfigPartHandler(handlers.Handler): @@ -53,6 +59,8 @@ class CloudConfigPartHandler(handlers.Handler): if self.file_names: file_lines.append("# from %s files" % (len(self.file_names))) for fn in self.file_names: + if not fn: + fn = '?' file_lines.append("# %s" % (fn)) file_lines.append("") if self.cloud_buf is not None: @@ -111,7 +119,10 @@ class CloudConfigPartHandler(handlers.Handler): return try: self._merge_part(payload, headers) - self.file_names.append(filename) + # Ensure filename is ok to store + for i in ("\n", "\r", "\t"): + filename = filename.replace(i, " ") + self.file_names.append(filename.strip()) except: util.logexc(LOG, "Failed at merging in cloud config part from %s", filename) diff --git a/tests/data/merge_sources/expected11.yaml b/tests/data/merge_sources/expected11.yaml new file mode 100644 index 00000000..c0530dc3 --- /dev/null +++ b/tests/data/merge_sources/expected11.yaml @@ -0,0 +1,5 @@ +#cloud-config + +a: 22 +b: 4 +c: 3 diff --git a/tests/data/merge_sources/expected12.yaml b/tests/data/merge_sources/expected12.yaml new file mode 100644 index 00000000..0421d2c8 --- /dev/null +++ b/tests/data/merge_sources/expected12.yaml @@ -0,0 +1,5 @@ +#cloud-config + +a: + e: + y: 2 diff --git a/tests/data/merge_sources/expected2.yaml b/tests/data/merge_sources/expected2.yaml index f5312eb1..6eccc2cf 100644 --- a/tests/data/merge_sources/expected2.yaml +++ b/tests/data/merge_sources/expected2.yaml @@ -1,3 +1,3 @@ -Blah: 1 +Blah: 3 Blah2: 2 -Blah3: 3 +Blah3: [1] diff --git a/tests/data/merge_sources/source11-1.yaml b/tests/data/merge_sources/source11-1.yaml new file mode 100644 index 00000000..ee29d681 --- /dev/null +++ b/tests/data/merge_sources/source11-1.yaml @@ -0,0 +1,5 @@ +#cloud-config + +a: 1 +b: 2 +c: 3 diff --git a/tests/data/merge_sources/source11-2.yaml b/tests/data/merge_sources/source11-2.yaml new file mode 100644 index 00000000..a9914c34 --- /dev/null +++ b/tests/data/merge_sources/source11-2.yaml @@ -0,0 +1,3 @@ +#cloud-config + +b: 4 diff --git a/tests/data/merge_sources/source11-3.yaml b/tests/data/merge_sources/source11-3.yaml new file mode 100644 index 00000000..8f2b8944 --- /dev/null +++ b/tests/data/merge_sources/source11-3.yaml @@ -0,0 +1,3 @@ +#cloud-config + +a: 22 diff --git a/tests/data/merge_sources/source12-1.yaml b/tests/data/merge_sources/source12-1.yaml new file mode 100644 index 00000000..09e7c899 --- /dev/null +++ b/tests/data/merge_sources/source12-1.yaml @@ -0,0 +1,8 @@ +#cloud-config + +a: + c: 1 + d: 2 + e: + z: a + y: b diff --git a/tests/data/merge_sources/source12-2.yaml b/tests/data/merge_sources/source12-2.yaml new file mode 100644 index 00000000..0421d2c8 --- /dev/null +++ b/tests/data/merge_sources/source12-2.yaml @@ -0,0 +1,5 @@ +#cloud-config + +a: + e: + y: 2 diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index dddf8c6c..ba1c67d7 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -167,6 +167,21 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): d = util.mergemanydict([a, b]) self.assertEquals(c, d) + def test_compat_merges_dict(self): + a = { + 'Blah': 1, + 'Blah2': 2, + 'Blah3': 3, + } + b = { + 'Blah': 1, + 'Blah2': 2, + 'Blah3': [1], + } + c = _old_mergedict(a, b) + d = util.mergemanydict([a, b]) + self.assertEquals(c, d) + def test_compat_merges_list(self): a = {'b': [1, 2, 3]} b = {'b': [4, 5]} diff --git a/tests/unittests/test_userdata.py b/tests/unittests/test_userdata.py index 5fb9acd9..0ebb0484 100644 --- a/tests/unittests/test_userdata.py +++ b/tests/unittests/test_userdata.py @@ -108,7 +108,7 @@ p: 1 contents = util.load_yaml(contents) self.assertEquals(contents['run'], ['b', 'c', 'stuff', 'morestuff']) self.assertEquals(contents['a'], 'be') - self.assertEquals(contents['e'], 'fg') + self.assertEquals(contents['e'], [1, 2, 3]) self.assertEquals(contents['p'], 1) def test_unhandled_type_warning(self): -- cgit v1.2.3