From e14d64a03c6aa3e567b57f0c0a003ca2185f4493 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 23 Apr 2013 22:53:53 -0700 Subject: Add a bunch of new merging test files + runner. --- tests/unittests/test_merging.py | 194 +++++++++++----------------------------- 1 file changed, 54 insertions(+), 140 deletions(-) (limited to 'tests/unittests') diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index ad137e85..470b18c7 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -1,142 +1,56 @@ from tests.unittests import helpers -from cloudinit import mergers - - -class TestSimpleRun(helpers.MockerTestCase): - def test_basic_merge(self): - source = { - 'Blah': ['blah2'], - 'Blah3': 'c', - } - merge_with = { - 'Blah2': ['blah3'], - 'Blah3': 'b', - 'Blah': ['123'], - } - # Basic merge should not do thing special - merge_how = "list()+dict()+str()" - merger_set = mergers.string_extract_mergers(merge_how) - self.assertEquals(3, len(merger_set)) - merger = mergers.construct(merger_set) - merged = merger.merge(source, merge_with) - self.assertEquals(merged['Blah'], ['blah2']) - self.assertEquals(merged['Blah2'], ['blah3']) - self.assertEquals(merged['Blah3'], 'c') - - def test_dict_overwrite(self): - source = { - 'Blah': ['blah2'], - } - merge_with = { - 'Blah': ['123'], - } - # Now lets try a dict overwrite - merge_how = "list()+dict(overwrite)+str()" - merger_set = mergers.string_extract_mergers(merge_how) - self.assertEquals(3, len(merger_set)) - merger = mergers.construct(merger_set) - merged = merger.merge(source, merge_with) - self.assertEquals(merged['Blah'], ['123']) - - def test_string_append(self): - source = { - 'Blah': 'blah2', - } - merge_with = { - 'Blah': '345', - } - merge_how = "list()+dict()+str(append)" - merger_set = mergers.string_extract_mergers(merge_how) - self.assertEquals(3, len(merger_set)) - merger = mergers.construct(merger_set) - merged = merger.merge(source, merge_with) - self.assertEquals(merged['Blah'], 'blah2345') - - def test_list_extend(self): - source = ['abc'] - merge_with = ['123'] - merge_how = "list(extend)+dict()+str()" - merger_set = mergers.string_extract_mergers(merge_how) - self.assertEquals(3, len(merger_set)) - merger = mergers.construct(merger_set) - merged = merger.merge(source, merge_with) - self.assertEquals(merged, ['abc', '123']) - - def test_deep_merge(self): - source = { - 'a': [1, 'b', 2], - 'b': 'blahblah', - 'c': { - 'e': [1, 2, 3], - 'f': 'bigblobof', - 'iamadict': { - 'ok': 'ok', - } - }, - 'run': [ - 'runme', - 'runme2', - ], - 'runmereally': [ - 'e', ['a'], 'd', - ], - } - merge_with = { - 'a': ['e', 'f', 'g'], - 'b': 'more', - 'c': { - 'a': 'b', - 'f': 'stuff', - }, - 'run': [ - 'morecmd', - 'moremoremore', - ], - 'runmereally': [ - 'blah', ['b'], 'e', - ], - } - merge_how = "list(extend)+dict()+str(append)" - merger_set = mergers.string_extract_mergers(merge_how) - self.assertEquals(3, len(merger_set)) - merger = mergers.construct(merger_set) - merged = merger.merge(source, merge_with) - self.assertEquals(merged['a'], [1, 'b', 2, 'e', 'f', 'g']) - self.assertEquals(merged['b'], 'blahblahmore') - self.assertEquals(merged['c']['f'], 'bigblobofstuff') - self.assertEquals(merged['run'], ['runme', 'runme2', 'morecmd', - 'moremoremore']) - self.assertEquals(merged['runmereally'], ['e', ['a'], 'd', 'blah', - ['b'], 'e']) - - def test_dict_overwrite_layered(self): - source = { - 'Blah3': { - 'f': '3', - 'g': { - 'a': 'b', - } - } - } - merge_with = { - 'Blah3': { - 'e': '2', - 'g': { - 'e': 'f', - } - } - } - merge_how = "list()+dict()+str()" - merger_set = mergers.string_extract_mergers(merge_how) - self.assertEquals(3, len(merger_set)) - merger = mergers.construct(merger_set) - merged = merger.merge(source, merge_with) - self.assertEquals(merged['Blah3'], { - 'e': '2', - 'f': '3', - 'g': { - 'a': 'b', - 'e': 'f', - } - }) +from cloudinit.handlers import cloud_config +from cloudinit.handlers import (CONTENT_START, CONTENT_END) + +from cloudinit import helpers as c_helpers +from cloudinit import util + +import collections +import glob +import os +import re + + +class TestSimpleRun(helpers.ResourceUsingTestCase): + def _load_merge_files(self, data_dir): + merge_root = self.resourceLocation(data_dir) + tests = [] + source_ids = collections.defaultdict(list) + expected_files = {} + for fn in glob.glob(os.path.join(merge_root, "source*.*yaml")): + base_fn = os.path.basename(fn) + file_id = re.match(r"source(\d+)\-(\d+)[.]yaml", base_fn) + if not file_id: + raise IOError("File %s does not have a numeric identifier" + % (fn)) + file_id = int(file_id.group(1)) + source_ids[file_id].append(fn) + expected_fn = os.path.join(merge_root, + "expected%s.yaml" % (file_id)) + if not os.path.isfile(expected_fn): + raise IOError("No expected file found at %s" % (expected_fn)) + expected_files[file_id] = expected_fn + for id in sorted(source_ids.keys()): + source_file_contents = [] + for fn in sorted(source_ids[id]): + source_file_contents.append(util.load_file(fn)) + expected = util.load_yaml(util.load_file(expected_files[id])) + tests.append((source_file_contents, expected)) + return tests + + def test_merge_samples(self): + tests = self._load_merge_files('merge_sources') + paths = c_helpers.Paths({}) + cc_handler = cloud_config.CloudConfigPartHandler(paths) + cc_handler.cloud_fn = None + for (payloads, expected_merge) in tests: + cc_handler.handle_part(None, CONTENT_START, None, + None, None, None) + for (i, p) in enumerate(payloads): + cc_handler.handle_part(None, None, "t-%s.yaml" % (i + 1), + p, None, {}) + merged_buf = cc_handler.cloud_buf + cc_handler.handle_part(None, CONTENT_END, None, + None, None, None) + self.assertEquals(expected_merge, merged_buf) -- cgit v1.2.3 From 4a669649b17cf01b6f89f7902b6683d02ef0bee1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 3 May 2013 14:41:28 -0700 Subject: More merging adjustments. Looks like this should be in pretty good shape and has passed some of the basic backwards compat. merging tests that I added. --- cloudinit/mergers/__init__.py | 2 +- cloudinit/mergers/m_dict.py | 27 ++++--- cloudinit/mergers/m_list.py | 9 +-- cloudinit/mergers/m_str.py | 44 ++++++++++++ tests/data/merge_sources/expected2.yaml | 4 +- tests/data/merge_sources/expected5.yaml | 7 ++ tests/data/merge_sources/source5-1.yaml | 6 ++ tests/data/merge_sources/source5-2.yaml | 8 +++ tests/unittests/test_merging.py | 123 +++++++++++++++++++++++++++----- tests/unittests/test_userdata.py | 5 +- 10 files changed, 201 insertions(+), 34 deletions(-) create mode 100644 cloudinit/mergers/m_str.py create mode 100644 tests/data/merge_sources/expected5.yaml create mode 100644 tests/data/merge_sources/source5-1.yaml create mode 100644 tests/data/merge_sources/source5-2.yaml (limited to 'tests/unittests') diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index 221e93b5..0978b2c6 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -25,7 +25,7 @@ from cloudinit import type_utils NAME_MTCH = re.compile(r"(^[a-zA-Z_][A-Za-z0-9_]*)\((.*?)\)$") LOG = logging.getLogger(__name__) -DEF_MERGE_TYPE = "list()+dict()" +DEF_MERGE_TYPE = "list()+dict()+str()" MERGER_PREFIX = 'm_' MERGER_ATTR = 'Merger' diff --git a/cloudinit/mergers/m_dict.py b/cloudinit/mergers/m_dict.py index 2c1c845f..82caa004 100644 --- a/cloudinit/mergers/m_dict.py +++ b/cloudinit/mergers/m_dict.py @@ -16,21 +16,32 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +DEF_MERGE_TYPE = 'no_replace' +MERGE_TYPES = ('replace', DEF_MERGE_TYPE,) + + +def _has_any(what, *keys): + for k in keys: + if k in what: + return True + return False + class Merger(object): def __init__(self, merger, opts): self._merger = merger # Affects merging behavior... - self._method = 'replace' - for m in ['replace', 'no_replace']: + self._method = DEF_MERGE_TYPE + for m in MERGE_TYPES: if m in opts: self._method = m break - # Affect how recursive merging is done on other primitives + # Affect how recursive merging is done on other primitives. self._recurse_str = 'recurse_str' in opts - self._recurse_dict = True - self._recurse_array = 'recurse_array' in opts + self._recurse_array = _has_any(opts, 'recurse_array', 'recurse_list') self._allow_delete = 'allow_delete' in opts + # Backwards compat require this to be on. + self._recurse_dict = True def __str__(self): s = ('DictMerger: (method=%s,recurse_str=%s,' @@ -42,14 +53,14 @@ class Merger(object): self._allow_delete) return s - def _do_dict_replace(self, value, merge_with, do_replace=True): + def _do_dict_replace(self, value, merge_with, do_replace): def merge_same_key(old_v, new_v): if do_replace: return new_v if isinstance(new_v, (list, tuple)) and self._recurse_array: return self._merger.merge(old_v, new_v) - if isinstance(new_v, (str, basestring)) and self._recurse_str: + if isinstance(new_v, (basestring)) and self._recurse_str: return self._merger.merge(old_v, new_v) if isinstance(new_v, (dict)) and self._recurse_dict: return self._merger.merge(old_v, new_v) @@ -70,7 +81,7 @@ class Merger(object): if not isinstance(merge_with, (dict)): return value if self._method == 'replace': - merged = self._do_dict_replace(dict(value), merge_with) + merged = self._do_dict_replace(dict(value), merge_with, True) elif self._method == 'no_replace': merged = self._do_dict_replace(dict(value), merge_with, False) else: diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index c6a23d85..50f279e8 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -32,10 +32,11 @@ class Merger(object): self._recurse_array = 'recurse_array' in opts def __str__(self): - return 'ListMerger: (m=%s,rs=%s,rd=%s,ra=%s)' % (self._method, - self._recurse_str, - self._recurse_dict, - self._recurse_array) + return ('ListMerger: (method=%s,recurse_str=%s,' + 'recurse_dict=%s,recurse_array=%s)') % (self._method, + self._recurse_str, + self._recurse_dict, + self._recurse_array) def _on_tuple(self, value, merge_with): return tuple(self._on_list(list(value), merge_with)) diff --git a/cloudinit/mergers/m_str.py b/cloudinit/mergers/m_str.py new file mode 100644 index 00000000..e22ce28a --- /dev/null +++ b/cloudinit/mergers/m_str.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class Merger(object): + def __init__(self, _merger, opts): + self._append = 'append' in opts + + def __str__(self): + return 'StringMerger: (append=%s)' % (self._append) + + # On encountering a unicode object to merge value with + # we will for now just proxy into the string method to let it handle it. + def _on_unicode(self, value, merge_with): + return self._on_str(value, merge_with) + + # On encountering a string object to merge with we will + # perform the following action, if appending we will + # merge them together, otherwise we will just return value. + def _on_str(self, value, merge_with): + if not isinstance(value, (basestring)): + return merge_with + if not self._append: + return merge_with + if isinstance(value, unicode): + return value + unicode(merge_with) + else: + return value + str(merge_with) diff --git a/tests/data/merge_sources/expected2.yaml b/tests/data/merge_sources/expected2.yaml index 6eccc2cf..f5312eb1 100644 --- a/tests/data/merge_sources/expected2.yaml +++ b/tests/data/merge_sources/expected2.yaml @@ -1,3 +1,3 @@ -Blah: 3 +Blah: 1 Blah2: 2 -Blah3: [1] +Blah3: 3 diff --git a/tests/data/merge_sources/expected5.yaml b/tests/data/merge_sources/expected5.yaml new file mode 100644 index 00000000..628f5878 --- /dev/null +++ b/tests/data/merge_sources/expected5.yaml @@ -0,0 +1,7 @@ +#cloud-config + +Blah: 3 +Blah2: 2 +Blah3: [1] + + diff --git a/tests/data/merge_sources/source5-1.yaml b/tests/data/merge_sources/source5-1.yaml new file mode 100644 index 00000000..c7a33aaa --- /dev/null +++ b/tests/data/merge_sources/source5-1.yaml @@ -0,0 +1,6 @@ +#cloud-config + + +Blah: 1 +Blah2: 2 +Blah3: 3 diff --git a/tests/data/merge_sources/source5-2.yaml b/tests/data/merge_sources/source5-2.yaml new file mode 100644 index 00000000..f61c96a2 --- /dev/null +++ b/tests/data/merge_sources/source5-2.yaml @@ -0,0 +1,8 @@ +#cloud-config + +Blah: 3 +Blah2: 2 +Blah3: [1] + + +merge_how: 'dict(replace)+list(append)' diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 470b18c7..f83522d7 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -11,14 +11,39 @@ import glob import os import re +SOURCE_PAT = "source*.*yaml" +EXPECTED_PAT = "expected%s.yaml" + + +def _old_mergedict(src, cand): + """ + Merge values from C{cand} into C{src}. + If C{src} has a key C{cand} will not override. + Nested dictionaries are merged recursively. + """ + if isinstance(src, dict) and isinstance(cand, dict): + for (k, v) in cand.iteritems(): + if k not in src: + src[k] = v + else: + src[k] = _old_mergedict(src[k], v) + return src + + +def _old_mergemanydict(*args): + out = {} + for a in args: + out = _old_mergedict(out, a) + return out + class TestSimpleRun(helpers.ResourceUsingTestCase): - def _load_merge_files(self, data_dir): - merge_root = self.resourceLocation(data_dir) + def _load_merge_files(self): + merge_root = self.resourceLocation('merge_sources') tests = [] source_ids = collections.defaultdict(list) expected_files = {} - for fn in glob.glob(os.path.join(merge_root, "source*.*yaml")): + for fn in glob.glob(os.path.join(merge_root, SOURCE_PAT)): base_fn = os.path.basename(fn) file_id = re.match(r"source(\d+)\-(\d+)[.]yaml", base_fn) if not file_id: @@ -26,31 +51,97 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): % (fn)) file_id = int(file_id.group(1)) source_ids[file_id].append(fn) - expected_fn = os.path.join(merge_root, - "expected%s.yaml" % (file_id)) + expected_fn = os.path.join(merge_root, EXPECTED_PAT % (file_id)) if not os.path.isfile(expected_fn): raise IOError("No expected file found at %s" % (expected_fn)) expected_files[file_id] = expected_fn - for id in sorted(source_ids.keys()): + for i in sorted(source_ids.keys()): source_file_contents = [] - for fn in sorted(source_ids[id]): - source_file_contents.append(util.load_file(fn)) - expected = util.load_yaml(util.load_file(expected_files[id])) - tests.append((source_file_contents, expected)) + for fn in sorted(source_ids[i]): + source_file_contents.append([fn, util.load_file(fn)]) + expected = util.load_yaml(util.load_file(expected_files[i])) + entry = [source_file_contents, [expected, expected_files[i]]] + tests.append(entry) return tests def test_merge_samples(self): - tests = self._load_merge_files('merge_sources') + tests = self._load_merge_files() paths = c_helpers.Paths({}) cc_handler = cloud_config.CloudConfigPartHandler(paths) cc_handler.cloud_fn = None - for (payloads, expected_merge) in tests: + for (payloads, (expected_merge, expected_fn)) in tests: cc_handler.handle_part(None, CONTENT_START, None, None, None, None) - for (i, p) in enumerate(payloads): - cc_handler.handle_part(None, None, "t-%s.yaml" % (i + 1), - p, None, {}) + merging_fns = [] + for (fn, contents) in payloads: + cc_handler.handle_part(None, None, "%s.yaml" % (fn), + contents, None, {}) + merging_fns.append(fn) merged_buf = cc_handler.cloud_buf cc_handler.handle_part(None, CONTENT_END, None, None, None, None) - self.assertEquals(expected_merge, merged_buf) + fail_msg = "Equality failure on checking %s with %s: %s != %s" + fail_msg = fail_msg % (expected_fn, + ",".join(merging_fns), merged_buf, + expected_merge) + self.assertEquals(expected_merge, merged_buf, msg=fail_msg) + + def test_compat_merges_dict(self): + a = { + '1': '2', + 'b': 'c', + } + b = { + 'b': 'e', + } + c = _old_mergedict(a, b) + d = util.mergemanydict([a, b]) + self.assertEquals(c, d) + + def test_compat_merges_list(self): + a = {'b': [1, 2, 3]} + b = {'b': [4, 5]} + c = {'b': [6, 7]} + e = _old_mergemanydict(a, b, c) + f = util.mergemanydict([a, b, c]) + self.assertEquals(e, f) + + def test_compat_merges_str(self): + a = {'b': "hi"} + b = {'b': "howdy"} + c = {'b': "hallo"} + e = _old_mergemanydict(a, b, c) + f = util.mergemanydict([a, b, c]) + self.assertEquals(e, f) + + def test_compat_merge_sub_dict(self): + a = { + '1': '2', + 'b': { + 'f': 'g', + } + } + b = { + 'b': { + 'e': 'c', + } + } + c = _old_mergedict(a, b) + d = util.mergemanydict([a, b]) + self.assertEquals(c, d) + + def test_compat_merge_sub_list(self): + a = { + '1': '2', + 'b': { + 'f': ['1'], + } + } + b = { + 'b': { + 'f': [], + } + } + c = _old_mergedict(a, b) + d = util.mergemanydict([a, b]) + self.assertEquals(c, d) diff --git a/tests/unittests/test_userdata.py b/tests/unittests/test_userdata.py index fdfe2542..5fb9acd9 100644 --- a/tests/unittests/test_userdata.py +++ b/tests/unittests/test_userdata.py @@ -60,7 +60,6 @@ run: - c ''' message1 = MIMEBase("text", "cloud-config") - message1['Merge-Type'] = 'dict()+list(extend)+str(append)' message1.set_payload(blob) blob2 = ''' @@ -72,7 +71,8 @@ run: - morestuff ''' message2 = MIMEBase("text", "cloud-config") - message2['X-Merge-Type'] = 'dict()+list(extend)+str()' + message2['X-Merge-Type'] = ('dict(recurse_array,' + 'recurse_str)+list(append)+str(append)') message2.set_payload(blob2) blob3 = ''' @@ -84,7 +84,6 @@ e: p: 1 ''' message3 = MIMEBase("text", "cloud-config") - message3['Merge-Type'] = 'dict()+list()+str()' message3.set_payload(blob3) messages = [message1, message2, message3] -- cgit v1.2.3 From c972396ecccb4b67eafc038a482ffeaa1df2c93e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 3 May 2013 14:46:52 -0700 Subject: A few more tests + cleanings. --- cloudinit/mergers/m_list.py | 7 +++++-- tests/unittests/test_merging.py | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) (limited to 'tests/unittests') diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index 50f279e8..34b32379 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -16,13 +16,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +DEF_MERGE_TYPE = 'replace' +MERGE_TYPES = ('append', 'prepend', DEF_MERGE_TYPE,) + class Merger(object): def __init__(self, merger, opts): self._merger = merger # Affects merging behavior... - self._method = 'replace' - for m in ['append', 'prepend', 'replace']: + self._method = DEF_MERGE_TYPE + for m in MERGE_TYPES: if m in opts: self._method = m break diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index f83522d7..cff8ac12 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -115,6 +115,30 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): self.assertEquals(e, f) def test_compat_merge_sub_dict(self): + a = { + '1': '2', + 'b': { + 'f': 'g', + 'e': 'c', + 'h': 'd', + 'hh': { + '1': 2, + }, + } + } + b = { + 'b': { + 'e': 'c', + 'hh': { + '3': 4, + } + } + } + c = _old_mergedict(a, b) + d = util.mergemanydict([a, b]) + self.assertEquals(c, d) + + def test_compat_merge_sub_dict2(self): a = { '1': '2', 'b': { -- cgit v1.2.3 From 5118a33b22a376954bd048c3142f2d3f7f55d003 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 3 May 2013 14:49:16 -0700 Subject: Rename the merge cc sample function name. --- tests/unittests/test_merging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/unittests') diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index cff8ac12..dd8c2eee 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -64,7 +64,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): tests.append(entry) return tests - def test_merge_samples(self): + def test_merge_cc_samples(self): tests = self._load_merge_files() paths = c_helpers.Paths({}) cc_handler = cloud_config.CloudConfigPartHandler(paths) -- cgit v1.2.3 From 9a1584b701cecbbba4a9371542114bcc806ec596 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 3 May 2013 15:05:45 -0700 Subject: A few pep8/pylint cleanups. --- cloudinit/mergers/m_dict.py | 7 ++----- cloudinit/mergers/m_list.py | 2 +- tests/unittests/test_merging.py | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) (limited to 'tests/unittests') diff --git a/cloudinit/mergers/m_dict.py b/cloudinit/mergers/m_dict.py index 82caa004..a16141fa 100644 --- a/cloudinit/mergers/m_dict.py +++ b/cloudinit/mergers/m_dict.py @@ -46,11 +46,8 @@ class Merger(object): def __str__(self): s = ('DictMerger: (method=%s,recurse_str=%s,' 'recurse_dict=%s,recurse_array=%s,allow_delete=%s)') - s = s % (self._method, - self._recurse_str, - self._recurse_dict, - self._recurse_array, - self._allow_delete) + s = s % (self._method, self._recurse_str, + self._recurse_dict, self._recurse_array, self._allow_delete) return s def _do_dict_replace(self, value, merge_with, do_replace): diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index 34b32379..8a0b5827 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -47,7 +47,7 @@ class Merger(object): def _on_list(self, value, merge_with): if (self._method == 'replace' and not isinstance(merge_with, (tuple, list))): - return merge_with + return merge_with # Ok we now know that what we are merging with is a list or tuple. merged_list = [] diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index dd8c2eee..ac2ccad4 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -12,7 +12,7 @@ import os import re SOURCE_PAT = "source*.*yaml" -EXPECTED_PAT = "expected%s.yaml" +EXPECTED_PAT = "expected%s.yaml" def _old_mergedict(src, cand): -- cgit v1.2.3 From 563af0754dc53fe4a95a4dee8ed18282f7a38104 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 4 May 2013 12:57:15 -0700 Subject: Add a set of randomized (seeded) dict tests. --- tests/unittests/test_merging.py | 69 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) (limited to 'tests/unittests') diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index ac2ccad4..dddf8c6c 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -9,10 +9,13 @@ from cloudinit import util import collections import glob import os +import random import re +import string SOURCE_PAT = "source*.*yaml" EXPECTED_PAT = "expected%s.yaml" +TYPES = [long, int, dict, str, list, tuple, None] def _old_mergedict(src, cand): @@ -37,6 +40,60 @@ def _old_mergemanydict(*args): return out +def _random_str(rand): + base = '' + for _i in xrange(rand.randint(1, 2**8)): + base += rand.choice(string.letters + string.digits) + return base + + +class _NoMoreException(Exception): + pass + + +def _make_dict(current_depth, max_depth, rand): + if current_depth >= max_depth: + raise _NoMoreException() + if current_depth == 0: + t = dict + else: + t = rand.choice(TYPES) + base = None + if t in [None]: + return base + if t in [dict, list, tuple]: + if t in [dict]: + amount = rand.randint(0, 5) + keys = [_random_str(rand) for _i in xrange(0, amount)] + base = {} + for k in keys: + try: + base[k] = _make_dict(current_depth + 1, max_depth, rand) + except _NoMoreException: + pass + elif t in [list, tuple]: + base = [] + amount = rand.randint(0, 5) + for _i in xrange(0, amount): + try: + base.append(_make_dict(current_depth + 1, max_depth, rand)) + except _NoMoreException: + pass + if t in [tuple]: + base = tuple(base) + elif t in [long, int]: + base = rand.randint(0, 2**8) + elif t in [str]: + base = _random_str(rand) + return base + + +def make_dict(max_depth, seed=None): + max_depth = max(1, max_depth) + rand = random.Random(seed) + return _make_dict(0, max_depth, rand) + + class TestSimpleRun(helpers.ResourceUsingTestCase): def _load_merge_files(self): merge_root = self.resourceLocation('merge_sources') @@ -64,6 +121,18 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): tests.append(entry) return tests + def test_seed_runs(self): + test_dicts = [] + for i in range(1, 50): + base_dicts = [] + for j in range(1, 50): + base_dicts.append(make_dict(5, i * j)) + test_dicts.append(base_dicts) + for test in test_dicts: + c = _old_mergemanydict(*test) + d = util.mergemanydict(test) + self.assertEquals(c, d) + def test_merge_cc_samples(self): tests = self._load_merge_files() paths = c_helpers.Paths({}) -- cgit v1.2.3 From 2b351c5435939d16ba06ec0c45847d47f4b21d51 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 9 May 2013 22:34:31 -0700 Subject: Fix the cloud config merging so that it is backwards compat. The new change for merging works well in the mergedict case but the default merging type for cloud config needs to reflect how yaml was loaded in bulk, which is the same as the replacing keys merging type that is now provided. --- cloudinit/handlers/cloud_config.py | 15 +++++++++++++-- tests/data/merge_sources/expected11.yaml | 5 +++++ tests/data/merge_sources/expected12.yaml | 5 +++++ tests/data/merge_sources/expected2.yaml | 4 ++-- tests/data/merge_sources/source11-1.yaml | 5 +++++ tests/data/merge_sources/source11-2.yaml | 3 +++ tests/data/merge_sources/source11-3.yaml | 3 +++ tests/data/merge_sources/source12-1.yaml | 8 ++++++++ tests/data/merge_sources/source12-2.yaml | 5 +++++ tests/unittests/test_merging.py | 15 +++++++++++++++ tests/unittests/test_userdata.py | 2 +- 11 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 tests/data/merge_sources/expected11.yaml create mode 100644 tests/data/merge_sources/expected12.yaml create mode 100644 tests/data/merge_sources/source11-1.yaml create mode 100644 tests/data/merge_sources/source11-2.yaml create mode 100644 tests/data/merge_sources/source11-3.yaml create mode 100644 tests/data/merge_sources/source12-1.yaml create mode 100644 tests/data/merge_sources/source12-2.yaml (limited to 'tests/unittests') diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 2ae9b226..529109ce 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -30,7 +30,13 @@ from cloudinit.settings import (PER_ALWAYS) LOG = logging.getLogger(__name__) MERGE_HEADER = 'Merge-Type' -DEF_MERGERS = mergers.default_mergers() + +# Due to the way the loading of yaml configuration was done previously, +# where previously each cloud config part was appended to a larger yaml +# file and then finally that file was loaded as one big yaml file we need +# to mimic that behavior by altering the default strategy to be replacing +# keys of later mergers. +DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()') class CloudConfigPartHandler(handlers.Handler): @@ -53,6 +59,8 @@ class CloudConfigPartHandler(handlers.Handler): if self.file_names: file_lines.append("# from %s files" % (len(self.file_names))) for fn in self.file_names: + if not fn: + fn = '?' file_lines.append("# %s" % (fn)) file_lines.append("") if self.cloud_buf is not None: @@ -111,7 +119,10 @@ class CloudConfigPartHandler(handlers.Handler): return try: self._merge_part(payload, headers) - self.file_names.append(filename) + # Ensure filename is ok to store + for i in ("\n", "\r", "\t"): + filename = filename.replace(i, " ") + self.file_names.append(filename.strip()) except: util.logexc(LOG, "Failed at merging in cloud config part from %s", filename) diff --git a/tests/data/merge_sources/expected11.yaml b/tests/data/merge_sources/expected11.yaml new file mode 100644 index 00000000..c0530dc3 --- /dev/null +++ b/tests/data/merge_sources/expected11.yaml @@ -0,0 +1,5 @@ +#cloud-config + +a: 22 +b: 4 +c: 3 diff --git a/tests/data/merge_sources/expected12.yaml b/tests/data/merge_sources/expected12.yaml new file mode 100644 index 00000000..0421d2c8 --- /dev/null +++ b/tests/data/merge_sources/expected12.yaml @@ -0,0 +1,5 @@ +#cloud-config + +a: + e: + y: 2 diff --git a/tests/data/merge_sources/expected2.yaml b/tests/data/merge_sources/expected2.yaml index f5312eb1..6eccc2cf 100644 --- a/tests/data/merge_sources/expected2.yaml +++ b/tests/data/merge_sources/expected2.yaml @@ -1,3 +1,3 @@ -Blah: 1 +Blah: 3 Blah2: 2 -Blah3: 3 +Blah3: [1] diff --git a/tests/data/merge_sources/source11-1.yaml b/tests/data/merge_sources/source11-1.yaml new file mode 100644 index 00000000..ee29d681 --- /dev/null +++ b/tests/data/merge_sources/source11-1.yaml @@ -0,0 +1,5 @@ +#cloud-config + +a: 1 +b: 2 +c: 3 diff --git a/tests/data/merge_sources/source11-2.yaml b/tests/data/merge_sources/source11-2.yaml new file mode 100644 index 00000000..a9914c34 --- /dev/null +++ b/tests/data/merge_sources/source11-2.yaml @@ -0,0 +1,3 @@ +#cloud-config + +b: 4 diff --git a/tests/data/merge_sources/source11-3.yaml b/tests/data/merge_sources/source11-3.yaml new file mode 100644 index 00000000..8f2b8944 --- /dev/null +++ b/tests/data/merge_sources/source11-3.yaml @@ -0,0 +1,3 @@ +#cloud-config + +a: 22 diff --git a/tests/data/merge_sources/source12-1.yaml b/tests/data/merge_sources/source12-1.yaml new file mode 100644 index 00000000..09e7c899 --- /dev/null +++ b/tests/data/merge_sources/source12-1.yaml @@ -0,0 +1,8 @@ +#cloud-config + +a: + c: 1 + d: 2 + e: + z: a + y: b diff --git a/tests/data/merge_sources/source12-2.yaml b/tests/data/merge_sources/source12-2.yaml new file mode 100644 index 00000000..0421d2c8 --- /dev/null +++ b/tests/data/merge_sources/source12-2.yaml @@ -0,0 +1,5 @@ +#cloud-config + +a: + e: + y: 2 diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index dddf8c6c..ba1c67d7 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -167,6 +167,21 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): d = util.mergemanydict([a, b]) self.assertEquals(c, d) + def test_compat_merges_dict(self): + a = { + 'Blah': 1, + 'Blah2': 2, + 'Blah3': 3, + } + b = { + 'Blah': 1, + 'Blah2': 2, + 'Blah3': [1], + } + c = _old_mergedict(a, b) + d = util.mergemanydict([a, b]) + self.assertEquals(c, d) + def test_compat_merges_list(self): a = {'b': [1, 2, 3]} b = {'b': [4, 5]} diff --git a/tests/unittests/test_userdata.py b/tests/unittests/test_userdata.py index 5fb9acd9..0ebb0484 100644 --- a/tests/unittests/test_userdata.py +++ b/tests/unittests/test_userdata.py @@ -108,7 +108,7 @@ p: 1 contents = util.load_yaml(contents) self.assertEquals(contents['run'], ['b', 'c', 'stuff', 'morestuff']) self.assertEquals(contents['a'], 'be') - self.assertEquals(contents['e'], 'fg') + self.assertEquals(contents['e'], [1, 2, 3]) self.assertEquals(contents['p'], 1) def test_unhandled_type_warning(self): -- cgit v1.2.3 From 4d83822985bc45dacd611859d76aa5cc3e35e3bc Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 10 May 2013 14:15:44 -0700 Subject: fix pep8 --- tests/unittests/test_merging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tests/unittests') diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index ba1c67d7..4c28f955 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -42,7 +42,7 @@ def _old_mergemanydict(*args): def _random_str(rand): base = '' - for _i in xrange(rand.randint(1, 2**8)): + for _i in xrange(rand.randint(1, 2 ** 8)): base += rand.choice(string.letters + string.digits) return base @@ -82,7 +82,7 @@ def _make_dict(current_depth, max_depth, rand): if t in [tuple]: base = tuple(base) elif t in [long, int]: - base = rand.randint(0, 2**8) + base = rand.randint(0, 2 ** 8) elif t in [str]: base = _random_str(rand) return base -- cgit v1.2.3 From 9f866ff5540558bab56f10e38481e4ad2efa079b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 10 May 2013 14:25:13 -0700 Subject: fix pylint --- tests/unittests/test_merging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tests/unittests') diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 4c28f955..486b9158 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -11,7 +11,7 @@ import glob import os import random import re -import string +import string # pylint: disable=W0402 SOURCE_PAT = "source*.*yaml" EXPECTED_PAT = "expected%s.yaml" @@ -167,7 +167,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): d = util.mergemanydict([a, b]) self.assertEquals(c, d) - def test_compat_merges_dict(self): + def test_compat_merges_dict2(self): a = { 'Blah': 1, 'Blah2': 2, -- cgit v1.2.3