From 0eabf9cdc2870982bcabc6e5d05c80078fa100cb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 18 Apr 2013 13:29:39 -0700 Subject: Adjust how merging is done. --- cloudinit/mergers/dict.py | 5 +++-- cloudinit/mergers/list.py | 18 +++--------------- 2 files changed, 6 insertions(+), 17 deletions(-) (limited to 'cloudinit/mergers') diff --git a/cloudinit/mergers/dict.py b/cloudinit/mergers/dict.py index 45a7d3a5..929d3865 100644 --- a/cloudinit/mergers/dict.py +++ b/cloudinit/mergers/dict.py @@ -20,7 +20,7 @@ class Merger(object): def __init__(self, merger, opts): self._merger = merger - self._overwrite = 'overwrite' in opts + self._not_overwrite = 'not_overwrite' in opts # This merging algorithm will attempt to merge with # another dictionary, on encountering any other type of object @@ -39,7 +39,8 @@ class Merger(object): merged = dict(value) for (k, v) in merge_with.items(): if k in merged: - if not self._overwrite: + if self._not_overwrite: + # Attempt to merge them.... merged[k] = self._merger.merge(merged[k], v) else: merged[k] = v diff --git a/cloudinit/mergers/list.py b/cloudinit/mergers/list.py index a56ff007..7c8b2e2a 100644 --- a/cloudinit/mergers/list.py +++ b/cloudinit/mergers/list.py @@ -20,7 +20,6 @@ class Merger(object): def __init__(self, merger, opts): self._merger = merger - self._discard_non = 'discard_non_list' in opts self._extend = 'extend' in opts def _on_tuple(self, value, merge_with): @@ -33,18 +32,7 @@ class Merger(object): # attempt to merge instead, which means that values from the list # to merge with will replace values in te original list (they will # also be merged recursively). - # - # If the value to merge with is not a list, and we are set to discared - # then no modifications will take place, otherwise we will just append - # the value to merge with onto the end of our own list. def _on_list(self, value, merge_with): - new_value = list(value) - if isinstance(merge_with, (tuple, list)): - if self._extend: - new_value.extend(merge_with) - else: - return new_value - else: - if not self._discard_non: - new_value.append(merge_with) - return new_value + if not self._extend or not isinstance(merge_with, (tuple, list)): + return merge_with + return list(value).extend(merge_with) -- cgit v1.2.3 From 50f91a1bca166b5e815a722aca573672b269bacb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 18 Apr 2013 14:15:37 -0700 Subject: Remove str merging for now. --- cloudinit/mergers/__init__.py | 2 +- cloudinit/mergers/m_str.py | 39 --------------------------------------- 2 files changed, 1 insertion(+), 40 deletions(-) delete mode 100644 cloudinit/mergers/m_str.py (limited to 'cloudinit/mergers') diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index f504e15f..2702496b 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -25,7 +25,7 @@ from cloudinit import type_utils NAME_MTCH = re.compile(r"(^[a-zA-Z_][A-Za-z0-9_]*)\((.*?)\)$") LOG = logging.getLogger(__name__) -DEF_MERGE_TYPE = "list()+dict()+str()" +DEF_MERGE_TYPE = "list()+dict()" MERGER_PREFIX = 'm_' MERGER_ATTR = 'Merger' diff --git a/cloudinit/mergers/m_str.py b/cloudinit/mergers/m_str.py deleted file mode 100644 index a0c57b53..00000000 --- a/cloudinit/mergers/m_str.py +++ /dev/null @@ -1,39 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class Merger(object): - def __init__(self, _merger, opts): - self._append = 'append' in opts - - # On encountering a unicode object to merge value with - # we will for now just proxy into the string method to let it handle it. - def _on_unicode(self, value, merge_with): - return self._on_str(value, merge_with) - - # On encountering a string object to merge with we will - # perform the following action, if appending we will - # merge them together, otherwise we will just return value. - def _on_str(self, value, merge_with): - if not self._append: - return merge_with - else: - if isinstance(value, (unicode)): - return value + unicode(merge_with) - else: - return value + str(merge_with) -- cgit v1.2.3 From 8441fe20fdd1d8bb195bc7d354c9e87d2f446ccd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 22 Apr 2013 12:43:35 -0700 Subject: Continue working on merging. --- cloudinit/handlers/cloud_config.py | 7 ++-- cloudinit/mergers/__init__.py | 12 +++++-- cloudinit/mergers/m_dict.py | 73 ++++++++++++++++++++++++++------------ cloudinit/mergers/m_list.py | 59 ++++++++++++++++++++++++------ 4 files changed, 112 insertions(+), 39 deletions(-) (limited to 'cloudinit/mergers') diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 7678a5b0..2ae9b226 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -85,17 +85,16 @@ class CloudConfigPartHandler(handlers.Handler): all_mergers.extend(mergers_header) if not all_mergers: all_mergers = DEF_MERGERS - return all_mergers + return (payload_yaml, all_mergers) def _merge_part(self, payload, headers): - my_mergers = self._extract_mergers(payload, headers) + (payload_yaml, my_mergers) = self._extract_mergers(payload, headers) LOG.debug("Merging by applying %s", my_mergers) merger = mergers.construct(my_mergers) if self.cloud_buf is None: # First time through, merge with an empty dict... self.cloud_buf = {} - self.cloud_buf = merger.merge(self.cloud_buf, - util.load_yaml(payload)) + self.cloud_buf = merger.merge(self.cloud_buf, payload_yaml) def _reset(self): self.file_names = [] diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index 2702496b..221e93b5 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -55,6 +55,9 @@ class UnknownMerger(object): if not meth: meth = self._handle_unknown args.insert(0, method_name) + LOG.debug("Merging '%s' into '%s' using method '%s' of '%s'", + type_name, type_utils.obj_name(merge_with), + meth.__name__, self) return meth(*args) @@ -66,6 +69,9 @@ class LookupMerger(UnknownMerger): else: self._lookups = lookups + def __str__(self): + return 'LookupMerger: (%s)' % (len(self._lookups)) + # For items which can not be merged by the parent this object # will lookup in a internally maintained set of objects and # find which one of those objects can perform the merge. If @@ -78,6 +84,8 @@ class LookupMerger(UnknownMerger): # First one that has that method/attr gets to be # the one that will be called meth = getattr(merger, meth_wanted) + LOG.debug(("Merging using located merger '%s'" + " since it had method '%s'"), merger, meth_wanted) break if not meth: return UnknownMerger._handle_unknown(self, meth_wanted, @@ -87,9 +95,9 @@ class LookupMerger(UnknownMerger): def dict_extract_mergers(config): parsed_mergers = [] - raw_mergers = config.get('merge_how') + raw_mergers = config.pop('merge_how', None) if raw_mergers is None: - raw_mergers = config.get('merge_type') + raw_mergers = config.pop('merge_type', None) if raw_mergers is None: return parsed_mergers if isinstance(raw_mergers, (str, basestring)): diff --git a/cloudinit/mergers/m_dict.py b/cloudinit/mergers/m_dict.py index 929d3865..2c1c845f 100644 --- a/cloudinit/mergers/m_dict.py +++ b/cloudinit/mergers/m_dict.py @@ -20,30 +20,59 @@ class Merger(object): def __init__(self, merger, opts): self._merger = merger - self._not_overwrite = 'not_overwrite' in opts + # Affects merging behavior... + self._method = 'replace' + for m in ['replace', 'no_replace']: + if m in opts: + self._method = m + break + # Affect how recursive merging is done on other primitives + self._recurse_str = 'recurse_str' in opts + self._recurse_dict = True + self._recurse_array = 'recurse_array' in opts + self._allow_delete = 'allow_delete' in opts + + def __str__(self): + s = ('DictMerger: (method=%s,recurse_str=%s,' + 'recurse_dict=%s,recurse_array=%s,allow_delete=%s)') + s = s % (self._method, + self._recurse_str, + self._recurse_dict, + self._recurse_array, + self._allow_delete) + return s + + def _do_dict_replace(self, value, merge_with, do_replace=True): + + def merge_same_key(old_v, new_v): + if do_replace: + return new_v + if isinstance(new_v, (list, tuple)) and self._recurse_array: + return self._merger.merge(old_v, new_v) + if isinstance(new_v, (str, basestring)) and self._recurse_str: + return self._merger.merge(old_v, new_v) + if isinstance(new_v, (dict)) and self._recurse_dict: + return self._merger.merge(old_v, new_v) + # Otherwise leave it be... + return old_v - # This merging algorithm will attempt to merge with - # another dictionary, on encountering any other type of object - # it will not merge with said object, but will instead return - # the original value - # - # On encountering a dictionary, it will create a new dictionary - # composed of the original and the one to merge with, if 'overwrite' - # is enabled then keys that exist in the original will be overwritten - # by keys in the one to merge with (and associated values). Otherwise - # if not in overwrite mode the 2 conflicting keys themselves will - # be merged. - def _on_dict(self, value, merge_with): - if not isinstance(merge_with, (dict)): - return value - merged = dict(value) for (k, v) in merge_with.items(): - if k in merged: - if self._not_overwrite: - # Attempt to merge them.... - merged[k] = self._merger.merge(merged[k], v) + if k in value: + if v is None and self._allow_delete: + value.pop(k) else: - merged[k] = v + value[k] = merge_same_key(value[k], v) else: - merged[k] = v + value[k] = v + return value + + def _on_dict(self, value, merge_with): + if not isinstance(merge_with, (dict)): + return value + if self._method == 'replace': + merged = self._do_dict_replace(dict(value), merge_with) + elif self._method == 'no_replace': + merged = self._do_dict_replace(dict(value), merge_with, False) + else: + raise NotImplementedError("Unknown merge type %s" % (self._method)) return merged diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index 208c5f52..c6a23d85 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -20,18 +20,55 @@ class Merger(object): def __init__(self, merger, opts): self._merger = merger - self._extend = 'extend' in opts + # Affects merging behavior... + self._method = 'replace' + for m in ['append', 'prepend', 'replace']: + if m in opts: + self._method = m + break + # Affect how recursive merging is done on other primitives + self._recurse_str = 'recurse_str' in opts + self._recurse_dict = 'recurse_dict' in opts + self._recurse_array = 'recurse_array' in opts + + def __str__(self): + return 'ListMerger: (m=%s,rs=%s,rd=%s,ra=%s)' % (self._method, + self._recurse_str, + self._recurse_dict, + self._recurse_array) def _on_tuple(self, value, merge_with): - return self._on_list(list(value), merge_with) + return tuple(self._on_list(list(value), merge_with)) - # On encountering a list or tuple type this action will be applied - # a new list will be returned, if the value to merge with is itself - # a list and we have been told to 'extend', then the value here will - # be extended with the other list. def _on_list(self, value, merge_with): - if not self._extend or not isinstance(merge_with, (tuple, list)): - return merge_with - # Leave the original list alone... - value = list(value) - return value.extend(merge_with) + if (self._method == 'replace' and + not isinstance(merge_with, (tuple, list))): + return merge_with + + # Ok we now know that what we are merging with is a list or tuple. + merged_list = [] + if self._method == 'prepend': + merged_list.extend(merge_with) + merged_list.extend(value) + return merged_list + elif self._method == 'append': + merged_list.extend(value) + merged_list.extend(merge_with) + return merged_list + + def merge_same_index(old_v, new_v): + if isinstance(new_v, (list, tuple)) and self._recurse_array: + return self._merger.merge(old_v, new_v) + if isinstance(new_v, (str, basestring)) and self._recurse_str: + return self._merger.merge(old_v, new_v) + if isinstance(new_v, (dict)) and self._recurse_dict: + return self._merger.merge(old_v, new_v) + # Otherwise leave it be... + return old_v + + # Ok now we are replacing same indexes + merged_list.extend(value) + common_len = min(len(merged_list), len(merge_with)) + for i in xrange(0, common_len): + merged_list[i] = merge_same_index(merged_list[i], merge_with[i]) + return merged_list -- cgit v1.2.3 From 4a669649b17cf01b6f89f7902b6683d02ef0bee1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 3 May 2013 14:41:28 -0700 Subject: More merging adjustments. Looks like this should be in pretty good shape and has passed some of the basic backwards compat. merging tests that I added. --- cloudinit/mergers/__init__.py | 2 +- cloudinit/mergers/m_dict.py | 27 ++++--- cloudinit/mergers/m_list.py | 9 +-- cloudinit/mergers/m_str.py | 44 ++++++++++++ tests/data/merge_sources/expected2.yaml | 4 +- tests/data/merge_sources/expected5.yaml | 7 ++ tests/data/merge_sources/source5-1.yaml | 6 ++ tests/data/merge_sources/source5-2.yaml | 8 +++ tests/unittests/test_merging.py | 123 +++++++++++++++++++++++++++----- tests/unittests/test_userdata.py | 5 +- 10 files changed, 201 insertions(+), 34 deletions(-) create mode 100644 cloudinit/mergers/m_str.py create mode 100644 tests/data/merge_sources/expected5.yaml create mode 100644 tests/data/merge_sources/source5-1.yaml create mode 100644 tests/data/merge_sources/source5-2.yaml (limited to 'cloudinit/mergers') diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index 221e93b5..0978b2c6 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -25,7 +25,7 @@ from cloudinit import type_utils NAME_MTCH = re.compile(r"(^[a-zA-Z_][A-Za-z0-9_]*)\((.*?)\)$") LOG = logging.getLogger(__name__) -DEF_MERGE_TYPE = "list()+dict()" +DEF_MERGE_TYPE = "list()+dict()+str()" MERGER_PREFIX = 'm_' MERGER_ATTR = 'Merger' diff --git a/cloudinit/mergers/m_dict.py b/cloudinit/mergers/m_dict.py index 2c1c845f..82caa004 100644 --- a/cloudinit/mergers/m_dict.py +++ b/cloudinit/mergers/m_dict.py @@ -16,21 +16,32 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +DEF_MERGE_TYPE = 'no_replace' +MERGE_TYPES = ('replace', DEF_MERGE_TYPE,) + + +def _has_any(what, *keys): + for k in keys: + if k in what: + return True + return False + class Merger(object): def __init__(self, merger, opts): self._merger = merger # Affects merging behavior... - self._method = 'replace' - for m in ['replace', 'no_replace']: + self._method = DEF_MERGE_TYPE + for m in MERGE_TYPES: if m in opts: self._method = m break - # Affect how recursive merging is done on other primitives + # Affect how recursive merging is done on other primitives. self._recurse_str = 'recurse_str' in opts - self._recurse_dict = True - self._recurse_array = 'recurse_array' in opts + self._recurse_array = _has_any(opts, 'recurse_array', 'recurse_list') self._allow_delete = 'allow_delete' in opts + # Backwards compat require this to be on. + self._recurse_dict = True def __str__(self): s = ('DictMerger: (method=%s,recurse_str=%s,' @@ -42,14 +53,14 @@ class Merger(object): self._allow_delete) return s - def _do_dict_replace(self, value, merge_with, do_replace=True): + def _do_dict_replace(self, value, merge_with, do_replace): def merge_same_key(old_v, new_v): if do_replace: return new_v if isinstance(new_v, (list, tuple)) and self._recurse_array: return self._merger.merge(old_v, new_v) - if isinstance(new_v, (str, basestring)) and self._recurse_str: + if isinstance(new_v, (basestring)) and self._recurse_str: return self._merger.merge(old_v, new_v) if isinstance(new_v, (dict)) and self._recurse_dict: return self._merger.merge(old_v, new_v) @@ -70,7 +81,7 @@ class Merger(object): if not isinstance(merge_with, (dict)): return value if self._method == 'replace': - merged = self._do_dict_replace(dict(value), merge_with) + merged = self._do_dict_replace(dict(value), merge_with, True) elif self._method == 'no_replace': merged = self._do_dict_replace(dict(value), merge_with, False) else: diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index c6a23d85..50f279e8 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -32,10 +32,11 @@ class Merger(object): self._recurse_array = 'recurse_array' in opts def __str__(self): - return 'ListMerger: (m=%s,rs=%s,rd=%s,ra=%s)' % (self._method, - self._recurse_str, - self._recurse_dict, - self._recurse_array) + return ('ListMerger: (method=%s,recurse_str=%s,' + 'recurse_dict=%s,recurse_array=%s)') % (self._method, + self._recurse_str, + self._recurse_dict, + self._recurse_array) def _on_tuple(self, value, merge_with): return tuple(self._on_list(list(value), merge_with)) diff --git a/cloudinit/mergers/m_str.py b/cloudinit/mergers/m_str.py new file mode 100644 index 00000000..e22ce28a --- /dev/null +++ b/cloudinit/mergers/m_str.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class Merger(object): + def __init__(self, _merger, opts): + self._append = 'append' in opts + + def __str__(self): + return 'StringMerger: (append=%s)' % (self._append) + + # On encountering a unicode object to merge value with + # we will for now just proxy into the string method to let it handle it. + def _on_unicode(self, value, merge_with): + return self._on_str(value, merge_with) + + # On encountering a string object to merge with we will + # perform the following action, if appending we will + # merge them together, otherwise we will just return value. + def _on_str(self, value, merge_with): + if not isinstance(value, (basestring)): + return merge_with + if not self._append: + return merge_with + if isinstance(value, unicode): + return value + unicode(merge_with) + else: + return value + str(merge_with) diff --git a/tests/data/merge_sources/expected2.yaml b/tests/data/merge_sources/expected2.yaml index 6eccc2cf..f5312eb1 100644 --- a/tests/data/merge_sources/expected2.yaml +++ b/tests/data/merge_sources/expected2.yaml @@ -1,3 +1,3 @@ -Blah: 3 +Blah: 1 Blah2: 2 -Blah3: [1] +Blah3: 3 diff --git a/tests/data/merge_sources/expected5.yaml b/tests/data/merge_sources/expected5.yaml new file mode 100644 index 00000000..628f5878 --- /dev/null +++ b/tests/data/merge_sources/expected5.yaml @@ -0,0 +1,7 @@ +#cloud-config + +Blah: 3 +Blah2: 2 +Blah3: [1] + + diff --git a/tests/data/merge_sources/source5-1.yaml b/tests/data/merge_sources/source5-1.yaml new file mode 100644 index 00000000..c7a33aaa --- /dev/null +++ b/tests/data/merge_sources/source5-1.yaml @@ -0,0 +1,6 @@ +#cloud-config + + +Blah: 1 +Blah2: 2 +Blah3: 3 diff --git a/tests/data/merge_sources/source5-2.yaml b/tests/data/merge_sources/source5-2.yaml new file mode 100644 index 00000000..f61c96a2 --- /dev/null +++ b/tests/data/merge_sources/source5-2.yaml @@ -0,0 +1,8 @@ +#cloud-config + +Blah: 3 +Blah2: 2 +Blah3: [1] + + +merge_how: 'dict(replace)+list(append)' diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 470b18c7..f83522d7 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -11,14 +11,39 @@ import glob import os import re +SOURCE_PAT = "source*.*yaml" +EXPECTED_PAT = "expected%s.yaml" + + +def _old_mergedict(src, cand): + """ + Merge values from C{cand} into C{src}. + If C{src} has a key C{cand} will not override. + Nested dictionaries are merged recursively. + """ + if isinstance(src, dict) and isinstance(cand, dict): + for (k, v) in cand.iteritems(): + if k not in src: + src[k] = v + else: + src[k] = _old_mergedict(src[k], v) + return src + + +def _old_mergemanydict(*args): + out = {} + for a in args: + out = _old_mergedict(out, a) + return out + class TestSimpleRun(helpers.ResourceUsingTestCase): - def _load_merge_files(self, data_dir): - merge_root = self.resourceLocation(data_dir) + def _load_merge_files(self): + merge_root = self.resourceLocation('merge_sources') tests = [] source_ids = collections.defaultdict(list) expected_files = {} - for fn in glob.glob(os.path.join(merge_root, "source*.*yaml")): + for fn in glob.glob(os.path.join(merge_root, SOURCE_PAT)): base_fn = os.path.basename(fn) file_id = re.match(r"source(\d+)\-(\d+)[.]yaml", base_fn) if not file_id: @@ -26,31 +51,97 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): % (fn)) file_id = int(file_id.group(1)) source_ids[file_id].append(fn) - expected_fn = os.path.join(merge_root, - "expected%s.yaml" % (file_id)) + expected_fn = os.path.join(merge_root, EXPECTED_PAT % (file_id)) if not os.path.isfile(expected_fn): raise IOError("No expected file found at %s" % (expected_fn)) expected_files[file_id] = expected_fn - for id in sorted(source_ids.keys()): + for i in sorted(source_ids.keys()): source_file_contents = [] - for fn in sorted(source_ids[id]): - source_file_contents.append(util.load_file(fn)) - expected = util.load_yaml(util.load_file(expected_files[id])) - tests.append((source_file_contents, expected)) + for fn in sorted(source_ids[i]): + source_file_contents.append([fn, util.load_file(fn)]) + expected = util.load_yaml(util.load_file(expected_files[i])) + entry = [source_file_contents, [expected, expected_files[i]]] + tests.append(entry) return tests def test_merge_samples(self): - tests = self._load_merge_files('merge_sources') + tests = self._load_merge_files() paths = c_helpers.Paths({}) cc_handler = cloud_config.CloudConfigPartHandler(paths) cc_handler.cloud_fn = None - for (payloads, expected_merge) in tests: + for (payloads, (expected_merge, expected_fn)) in tests: cc_handler.handle_part(None, CONTENT_START, None, None, None, None) - for (i, p) in enumerate(payloads): - cc_handler.handle_part(None, None, "t-%s.yaml" % (i + 1), - p, None, {}) + merging_fns = [] + for (fn, contents) in payloads: + cc_handler.handle_part(None, None, "%s.yaml" % (fn), + contents, None, {}) + merging_fns.append(fn) merged_buf = cc_handler.cloud_buf cc_handler.handle_part(None, CONTENT_END, None, None, None, None) - self.assertEquals(expected_merge, merged_buf) + fail_msg = "Equality failure on checking %s with %s: %s != %s" + fail_msg = fail_msg % (expected_fn, + ",".join(merging_fns), merged_buf, + expected_merge) + self.assertEquals(expected_merge, merged_buf, msg=fail_msg) + + def test_compat_merges_dict(self): + a = { + '1': '2', + 'b': 'c', + } + b = { + 'b': 'e', + } + c = _old_mergedict(a, b) + d = util.mergemanydict([a, b]) + self.assertEquals(c, d) + + def test_compat_merges_list(self): + a = {'b': [1, 2, 3]} + b = {'b': [4, 5]} + c = {'b': [6, 7]} + e = _old_mergemanydict(a, b, c) + f = util.mergemanydict([a, b, c]) + self.assertEquals(e, f) + + def test_compat_merges_str(self): + a = {'b': "hi"} + b = {'b': "howdy"} + c = {'b': "hallo"} + e = _old_mergemanydict(a, b, c) + f = util.mergemanydict([a, b, c]) + self.assertEquals(e, f) + + def test_compat_merge_sub_dict(self): + a = { + '1': '2', + 'b': { + 'f': 'g', + } + } + b = { + 'b': { + 'e': 'c', + } + } + c = _old_mergedict(a, b) + d = util.mergemanydict([a, b]) + self.assertEquals(c, d) + + def test_compat_merge_sub_list(self): + a = { + '1': '2', + 'b': { + 'f': ['1'], + } + } + b = { + 'b': { + 'f': [], + } + } + c = _old_mergedict(a, b) + d = util.mergemanydict([a, b]) + self.assertEquals(c, d) diff --git a/tests/unittests/test_userdata.py b/tests/unittests/test_userdata.py index fdfe2542..5fb9acd9 100644 --- a/tests/unittests/test_userdata.py +++ b/tests/unittests/test_userdata.py @@ -60,7 +60,6 @@ run: - c ''' message1 = MIMEBase("text", "cloud-config") - message1['Merge-Type'] = 'dict()+list(extend)+str(append)' message1.set_payload(blob) blob2 = ''' @@ -72,7 +71,8 @@ run: - morestuff ''' message2 = MIMEBase("text", "cloud-config") - message2['X-Merge-Type'] = 'dict()+list(extend)+str()' + message2['X-Merge-Type'] = ('dict(recurse_array,' + 'recurse_str)+list(append)+str(append)') message2.set_payload(blob2) blob3 = ''' @@ -84,7 +84,6 @@ e: p: 1 ''' message3 = MIMEBase("text", "cloud-config") - message3['Merge-Type'] = 'dict()+list()+str()' message3.set_payload(blob3) messages = [message1, message2, message3] -- cgit v1.2.3 From c972396ecccb4b67eafc038a482ffeaa1df2c93e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 3 May 2013 14:46:52 -0700 Subject: A few more tests + cleanings. --- cloudinit/mergers/m_list.py | 7 +++++-- tests/unittests/test_merging.py | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) (limited to 'cloudinit/mergers') diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index 50f279e8..34b32379 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -16,13 +16,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +DEF_MERGE_TYPE = 'replace' +MERGE_TYPES = ('append', 'prepend', DEF_MERGE_TYPE,) + class Merger(object): def __init__(self, merger, opts): self._merger = merger # Affects merging behavior... - self._method = 'replace' - for m in ['append', 'prepend', 'replace']: + self._method = DEF_MERGE_TYPE + for m in MERGE_TYPES: if m in opts: self._method = m break diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index f83522d7..cff8ac12 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -115,6 +115,30 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): self.assertEquals(e, f) def test_compat_merge_sub_dict(self): + a = { + '1': '2', + 'b': { + 'f': 'g', + 'e': 'c', + 'h': 'd', + 'hh': { + '1': 2, + }, + } + } + b = { + 'b': { + 'e': 'c', + 'hh': { + '3': 4, + } + } + } + c = _old_mergedict(a, b) + d = util.mergemanydict([a, b]) + self.assertEquals(c, d) + + def test_compat_merge_sub_dict2(self): a = { '1': '2', 'b': { -- cgit v1.2.3 From 9a1584b701cecbbba4a9371542114bcc806ec596 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 3 May 2013 15:05:45 -0700 Subject: A few pep8/pylint cleanups. --- cloudinit/mergers/m_dict.py | 7 ++----- cloudinit/mergers/m_list.py | 2 +- tests/unittests/test_merging.py | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) (limited to 'cloudinit/mergers') diff --git a/cloudinit/mergers/m_dict.py b/cloudinit/mergers/m_dict.py index 82caa004..a16141fa 100644 --- a/cloudinit/mergers/m_dict.py +++ b/cloudinit/mergers/m_dict.py @@ -46,11 +46,8 @@ class Merger(object): def __str__(self): s = ('DictMerger: (method=%s,recurse_str=%s,' 'recurse_dict=%s,recurse_array=%s,allow_delete=%s)') - s = s % (self._method, - self._recurse_str, - self._recurse_dict, - self._recurse_array, - self._allow_delete) + s = s % (self._method, self._recurse_str, + self._recurse_dict, self._recurse_array, self._allow_delete) return s def _do_dict_replace(self, value, merge_with, do_replace): diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index 34b32379..8a0b5827 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -47,7 +47,7 @@ class Merger(object): def _on_list(self, value, merge_with): if (self._method == 'replace' and not isinstance(merge_with, (tuple, list))): - return merge_with + return merge_with # Ok we now know that what we are merging with is a list or tuple. merged_list = [] diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index dd8c2eee..ac2ccad4 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -12,7 +12,7 @@ import os import re SOURCE_PAT = "source*.*yaml" -EXPECTED_PAT = "expected%s.yaml" +EXPECTED_PAT = "expected%s.yaml" def _old_mergedict(src, cand): -- cgit v1.2.3 From 229df67191e7dff058151f1e1f6e007667d55d9c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 7 May 2013 21:38:06 -0700 Subject: A few more test files. --- cloudinit/mergers/m_list.py | 12 ++++++++--- tests/data/merge_sources/expected6.yaml | 9 ++++++++ tests/data/merge_sources/expected7.yaml | 38 +++++++++++++++++++++++++++++++++ tests/data/merge_sources/expected8.yaml | 7 ++++++ tests/data/merge_sources/expected9.yaml | 5 +++++ tests/data/merge_sources/source6-1.yaml | 5 +++++ tests/data/merge_sources/source6-2.yaml | 8 +++++++ tests/data/merge_sources/source7-1.yaml | 27 +++++++++++++++++++++++ tests/data/merge_sources/source7-2.yaml | 17 +++++++++++++++ tests/data/merge_sources/source8-1.yaml | 7 ++++++ tests/data/merge_sources/source8-2.yaml | 6 ++++++ tests/data/merge_sources/source9-1.yaml | 5 +++++ tests/data/merge_sources/source9-2.yaml | 6 ++++++ 13 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 tests/data/merge_sources/expected6.yaml create mode 100644 tests/data/merge_sources/expected7.yaml create mode 100644 tests/data/merge_sources/expected8.yaml create mode 100644 tests/data/merge_sources/expected9.yaml create mode 100644 tests/data/merge_sources/source6-1.yaml create mode 100644 tests/data/merge_sources/source6-2.yaml create mode 100644 tests/data/merge_sources/source7-1.yaml create mode 100644 tests/data/merge_sources/source7-2.yaml create mode 100644 tests/data/merge_sources/source8-1.yaml create mode 100644 tests/data/merge_sources/source8-2.yaml create mode 100644 tests/data/merge_sources/source9-1.yaml create mode 100644 tests/data/merge_sources/source9-2.yaml (limited to 'cloudinit/mergers') diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index 8a0b5827..1184ded7 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -19,6 +19,12 @@ DEF_MERGE_TYPE = 'replace' MERGE_TYPES = ('append', 'prepend', DEF_MERGE_TYPE,) +def _has_any(what, *keys): + for k in keys: + if k in what: + return True + return False + class Merger(object): def __init__(self, merger, opts): @@ -30,9 +36,9 @@ class Merger(object): self._method = m break # Affect how recursive merging is done on other primitives - self._recurse_str = 'recurse_str' in opts - self._recurse_dict = 'recurse_dict' in opts - self._recurse_array = 'recurse_array' in opts + self._recurse_str = _has_any(opts, 'recurse_str') + self._recurse_dict = _has_any(opts, 'recurse_dict') + self._recurse_array = _has_any(opts, 'recurse_array', 'recurse_list') def __str__(self): return ('ListMerger: (method=%s,recurse_str=%s,' diff --git a/tests/data/merge_sources/expected6.yaml b/tests/data/merge_sources/expected6.yaml new file mode 100644 index 00000000..7afe1d7c --- /dev/null +++ b/tests/data/merge_sources/expected6.yaml @@ -0,0 +1,9 @@ +#cloud-config + +run_cmds: + - bash + - top + - ps + - vi + - emacs + diff --git a/tests/data/merge_sources/expected7.yaml b/tests/data/merge_sources/expected7.yaml new file mode 100644 index 00000000..25284f04 --- /dev/null +++ b/tests/data/merge_sources/expected7.yaml @@ -0,0 +1,38 @@ +#cloud-config + +users: + - default + - name: foobar + gecos: Foo B. Bar + primary-group: foobar + groups: users + selinux-user: staff_u + expiredate: 2012-09-01 + ssh-import-id: foobar + lock-passwd: false + passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ + - name: barfoo + gecos: Bar B. Foo + sudo: ALL=(ALL) NOPASSWD:ALL + groups: users, admin + ssh-import-id: None + lock-passwd: true + ssh-authorized-keys: + - + - + - name: cloudy + gecos: Magic Cloud App Daemon User + inactive: true + system: true + - bob + - joe + - sue + - name: foobar_jr + gecos: Foo B. Bar Jr + primary-group: foobar + groups: users + selinux-user: staff_u + expiredate: 2012-09-01 + ssh-import-id: foobar + lock-passwd: false + passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ diff --git a/tests/data/merge_sources/expected8.yaml b/tests/data/merge_sources/expected8.yaml new file mode 100644 index 00000000..69ca562d --- /dev/null +++ b/tests/data/merge_sources/expected8.yaml @@ -0,0 +1,7 @@ +#cloud-config + +mounts: + - [ ephemeral22, /mnt, auto, "defaults,noexec" ] + - [ sdc, /opt/data ] + - [ xvdh, /opt/data, "auto", "defaults,nobootwait", "0", "0" ] + - [ dd, /dev/zero ] diff --git a/tests/data/merge_sources/expected9.yaml b/tests/data/merge_sources/expected9.yaml new file mode 100644 index 00000000..00f91ca0 --- /dev/null +++ b/tests/data/merge_sources/expected9.yaml @@ -0,0 +1,5 @@ +#cloud-config + +phone_home: + url: http://my.example.com/$INSTANCE_ID/$BLAH_BLAH + post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id ] diff --git a/tests/data/merge_sources/source6-1.yaml b/tests/data/merge_sources/source6-1.yaml new file mode 100644 index 00000000..519f7309 --- /dev/null +++ b/tests/data/merge_sources/source6-1.yaml @@ -0,0 +1,5 @@ +#cloud-config + +run_cmds: + - bash + - top diff --git a/tests/data/merge_sources/source6-2.yaml b/tests/data/merge_sources/source6-2.yaml new file mode 100644 index 00000000..d8fac446 --- /dev/null +++ b/tests/data/merge_sources/source6-2.yaml @@ -0,0 +1,8 @@ +#cloud-config + +run_cmds: + - ps + - vi + - emacs + +merge_type: 'list(append)+dict(recurse_array)+str()' diff --git a/tests/data/merge_sources/source7-1.yaml b/tests/data/merge_sources/source7-1.yaml new file mode 100644 index 00000000..8fb9b32a --- /dev/null +++ b/tests/data/merge_sources/source7-1.yaml @@ -0,0 +1,27 @@ +#cloud-config + +users: + - default + - name: foobar + gecos: Foo B. Bar + primary-group: foobar + groups: users + selinux-user: staff_u + expiredate: 2012-09-01 + ssh-import-id: foobar + lock-passwd: false + passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ + - name: barfoo + gecos: Bar B. Foo + sudo: ALL=(ALL) NOPASSWD:ALL + groups: users, admin + ssh-import-id: None + lock-passwd: true + ssh-authorized-keys: + - + - + - name: cloudy + gecos: Magic Cloud App Daemon User + inactive: true + system: true + diff --git a/tests/data/merge_sources/source7-2.yaml b/tests/data/merge_sources/source7-2.yaml new file mode 100644 index 00000000..1e26201b --- /dev/null +++ b/tests/data/merge_sources/source7-2.yaml @@ -0,0 +1,17 @@ +#cloud-config + +users: + - bob + - joe + - sue + - name: foobar_jr + gecos: Foo B. Bar Jr + primary-group: foobar + groups: users + selinux-user: staff_u + expiredate: 2012-09-01 + ssh-import-id: foobar + lock-passwd: false + passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ + +merge_how: "dict(recurse_array)+list(append)" diff --git a/tests/data/merge_sources/source8-1.yaml b/tests/data/merge_sources/source8-1.yaml new file mode 100644 index 00000000..5ea51c2c --- /dev/null +++ b/tests/data/merge_sources/source8-1.yaml @@ -0,0 +1,7 @@ +#cloud-config + +mounts: + - [ ephemeral0, /mnt, auto, "defaults,noexec" ] + - [ sdc, /opt/data ] + - [ xvdh, /opt/data, "auto", "defaults,nobootwait", "0", "0" ] + - [ dd, /dev/zero ] diff --git a/tests/data/merge_sources/source8-2.yaml b/tests/data/merge_sources/source8-2.yaml new file mode 100644 index 00000000..7fa3262b --- /dev/null +++ b/tests/data/merge_sources/source8-2.yaml @@ -0,0 +1,6 @@ +#cloud-config + +mounts: + - [ ephemeral22, /mnt, auto, "defaults,noexec" ] + +merge_how: 'dict(recurse_array)+list(recurse_list,recurse_str)+str()' diff --git a/tests/data/merge_sources/source9-1.yaml b/tests/data/merge_sources/source9-1.yaml new file mode 100644 index 00000000..0b102ba6 --- /dev/null +++ b/tests/data/merge_sources/source9-1.yaml @@ -0,0 +1,5 @@ +#cloud-config + +phone_home: + url: http://my.example.com/$INSTANCE_ID/ + post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id ] diff --git a/tests/data/merge_sources/source9-2.yaml b/tests/data/merge_sources/source9-2.yaml new file mode 100644 index 00000000..ac85afc6 --- /dev/null +++ b/tests/data/merge_sources/source9-2.yaml @@ -0,0 +1,6 @@ +#cloud-config + +phone_home: + url: $BLAH_BLAH + +merge_how: 'dict(recurse_str)+str(append)' -- cgit v1.2.3 From 670b46d151477d32056f3fa4eb968c7960f3b472 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 9 May 2013 23:04:03 -0700 Subject: Allow lists to have no_replace option. --- cloudinit/mergers/m_list.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'cloudinit/mergers') diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index 1184ded7..76591bea 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -17,7 +17,7 @@ # along with this program. If not, see . DEF_MERGE_TYPE = 'replace' -MERGE_TYPES = ('append', 'prepend', DEF_MERGE_TYPE,) +MERGE_TYPES = ('append', 'prepend', DEF_MERGE_TYPE, 'no_replace') def _has_any(what, *keys): for k in keys: @@ -67,14 +67,16 @@ class Merger(object): return merged_list def merge_same_index(old_v, new_v): + if self._method == 'no_replace': + # Leave it be... + return old_v if isinstance(new_v, (list, tuple)) and self._recurse_array: return self._merger.merge(old_v, new_v) if isinstance(new_v, (str, basestring)) and self._recurse_str: return self._merger.merge(old_v, new_v) if isinstance(new_v, (dict)) and self._recurse_dict: return self._merger.merge(old_v, new_v) - # Otherwise leave it be... - return old_v + return new_v # Ok now we are replacing same indexes merged_list.extend(value) -- cgit v1.2.3