From 6ad068d1ae175d784481fe8f8e190b2721a221f5 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Wed, 17 Apr 2013 10:17:09 -0700
Subject: Altering the order of merging.

---
 cloudinit/handlers/cloud_config.py | 12 +++---------
 cloudinit/util.py                  | 23 +++++++++--------------
 2 files changed, 12 insertions(+), 23 deletions(-)

diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py
index d30d6338..7678a5b0 100644
--- a/cloudinit/handlers/cloud_config.py
+++ b/cloudinit/handlers/cloud_config.py
@@ -39,7 +39,6 @@ class CloudConfigPartHandler(handlers.Handler):
         self.cloud_buf = None
         self.cloud_fn = paths.get_ipath("cloud_config")
         self.file_names = []
-        self.mergers = [DEF_MERGERS]
 
     def list_types(self):
         return [
@@ -89,13 +88,9 @@ class CloudConfigPartHandler(handlers.Handler):
         return all_mergers
 
     def _merge_part(self, payload, headers):
-        next_mergers = self._extract_mergers(payload, headers)
-        # Use the merger list from the last call, since it is the one
-        # that will be defining how to merge with the next payload.
-        curr_mergers = list(self.mergers[-1])
-        LOG.debug("Merging by applying %s", curr_mergers)
-        self.mergers.append(next_mergers)
-        merger = mergers.construct(curr_mergers)
+        my_mergers = self._extract_mergers(payload, headers)
+        LOG.debug("Merging by applying %s", my_mergers)
+        merger = mergers.construct(my_mergers)
         if self.cloud_buf is None:
             # First time through, merge with an empty dict...
             self.cloud_buf = {}
@@ -105,7 +100,6 @@ class CloudConfigPartHandler(handlers.Handler):
     def _reset(self):
         self.file_names = []
         self.cloud_buf = None
-        self.mergers = [DEF_MERGERS]
 
     def handle_part(self, _data, ctype, filename,  # pylint: disable=W0221
                     payload, _frequency, headers):  # pylint: disable=W0613
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 50de55fe..f7ff28cc 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -543,21 +543,16 @@ def make_url(scheme, host, port=None,
 def mergemanydict(srcs, reverse=False):
     if reverse:
         srcs = reversed(srcs)
-    m_cfg = {}
-    merge_how = [mergers.default_mergers()]
-    for a_cfg in srcs:
-        if a_cfg:
-            # Take the last merger as the one that
-            # will define how to merge next...
-            mergers_to_apply = list(merge_how[-1])
+    merged_cfg = {}
+    for cfg in srcs:
+        if cfg:
+            # Figure out which mergers to apply...
+            mergers_to_apply = mergers.dict_extract_mergers(cfg)
+            if not mergers_to_apply:
+                mergers_to_apply = mergers.default_mergers()
             merger = mergers.construct(mergers_to_apply)
-            m_cfg = merger.merge(m_cfg, a_cfg)
-            # If the config has now has new merger set,
-            # extract them to be used next time...
-            new_mergers = mergers.dict_extract_mergers(m_cfg)
-            if new_mergers:
-                merge_how.append(new_mergers)
-    return m_cfg
+            merged_cfg = merger.merge(merged_cfg, cfg)
+    return merged_cfg
 
 
 @contextlib.contextmanager
-- 
cgit v1.2.3


From 0eabf9cdc2870982bcabc6e5d05c80078fa100cb Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Thu, 18 Apr 2013 13:29:39 -0700
Subject: Adjust how merging is done.

---
 cloudinit/mergers/dict.py |  5 +++--
 cloudinit/mergers/list.py | 18 +++---------------
 2 files changed, 6 insertions(+), 17 deletions(-)

diff --git a/cloudinit/mergers/dict.py b/cloudinit/mergers/dict.py
index 45a7d3a5..929d3865 100644
--- a/cloudinit/mergers/dict.py
+++ b/cloudinit/mergers/dict.py
@@ -20,7 +20,7 @@
 class Merger(object):
     def __init__(self, merger, opts):
         self._merger = merger
-        self._overwrite = 'overwrite' in opts
+        self._not_overwrite = 'not_overwrite' in opts
 
     # This merging algorithm will attempt to merge with
     # another dictionary, on encountering any other type of object
@@ -39,7 +39,8 @@ class Merger(object):
         merged = dict(value)
         for (k, v) in merge_with.items():
             if k in merged:
-                if not self._overwrite:
+                if self._not_overwrite:
+                    # Attempt to merge them....
                     merged[k] = self._merger.merge(merged[k], v)
                 else:
                     merged[k] = v
diff --git a/cloudinit/mergers/list.py b/cloudinit/mergers/list.py
index a56ff007..7c8b2e2a 100644
--- a/cloudinit/mergers/list.py
+++ b/cloudinit/mergers/list.py
@@ -20,7 +20,6 @@
 class Merger(object):
     def __init__(self, merger, opts):
         self._merger = merger
-        self._discard_non = 'discard_non_list' in opts
         self._extend = 'extend' in opts
 
     def _on_tuple(self, value, merge_with):
@@ -33,18 +32,7 @@ class Merger(object):
     # attempt to merge instead, which means that values from the list
     # to merge with will replace values in te original list (they will
     # also be merged recursively).
-    #
-    # If the value to merge with is not a list, and we are set to discared
-    # then no modifications will take place, otherwise we will just append
-    # the value to merge with onto the end of our own list.
     def _on_list(self, value, merge_with):
-        new_value = list(value)
-        if isinstance(merge_with, (tuple, list)):
-            if self._extend:
-                new_value.extend(merge_with)
-            else:
-                return new_value
-        else:
-            if not self._discard_non:
-                new_value.append(merge_with)
-        return new_value
+        if not self._extend or not isinstance(merge_with, (tuple, list)):
+            return merge_with
+        return list(value).extend(merge_with)
-- 
cgit v1.2.3


From 50f91a1bca166b5e815a722aca573672b269bacb Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Thu, 18 Apr 2013 14:15:37 -0700
Subject: Remove str merging for now.

---
 cloudinit/mergers/__init__.py |  2 +-
 cloudinit/mergers/m_str.py    | 39 ---------------------------------------
 2 files changed, 1 insertion(+), 40 deletions(-)
 delete mode 100644 cloudinit/mergers/m_str.py

diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py
index f504e15f..2702496b 100644
--- a/cloudinit/mergers/__init__.py
+++ b/cloudinit/mergers/__init__.py
@@ -25,7 +25,7 @@ from cloudinit import type_utils
 NAME_MTCH = re.compile(r"(^[a-zA-Z_][A-Za-z0-9_]*)\((.*?)\)$")
 
 LOG = logging.getLogger(__name__)
-DEF_MERGE_TYPE = "list()+dict()+str()"
+DEF_MERGE_TYPE = "list()+dict()"
 MERGER_PREFIX = 'm_'
 MERGER_ATTR = 'Merger'
 
diff --git a/cloudinit/mergers/m_str.py b/cloudinit/mergers/m_str.py
deleted file mode 100644
index a0c57b53..00000000
--- a/cloudinit/mergers/m_str.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# vi: ts=4 expandtab
-#
-#    Copyright (C) 2012 Yahoo! Inc.
-#
-#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License version 3, as
-#    published by the Free Software Foundation.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-
-class Merger(object):
-    def __init__(self, _merger, opts):
-        self._append = 'append' in opts
-
-    # On encountering a unicode object to merge value with
-    # we will for now just proxy into the string method to let it handle it.
-    def _on_unicode(self, value, merge_with):
-        return self._on_str(value, merge_with)
-
-    # On encountering a string object to merge with we will
-    # perform the following action, if appending we will
-    # merge them together, otherwise we will just return value.
-    def _on_str(self, value, merge_with):
-        if not self._append:
-            return merge_with
-        else:
-            if isinstance(value, (unicode)):
-                return value + unicode(merge_with)
-            else:
-                return value + str(merge_with)
-- 
cgit v1.2.3


From 8441fe20fdd1d8bb195bc7d354c9e87d2f446ccd Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Mon, 22 Apr 2013 12:43:35 -0700
Subject: Continue working on merging.

---
 cloudinit/handlers/cloud_config.py |  7 ++--
 cloudinit/mergers/__init__.py      | 12 +++++--
 cloudinit/mergers/m_dict.py        | 73 ++++++++++++++++++++++++++------------
 cloudinit/mergers/m_list.py        | 59 ++++++++++++++++++++++++------
 4 files changed, 112 insertions(+), 39 deletions(-)

diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py
index 7678a5b0..2ae9b226 100644
--- a/cloudinit/handlers/cloud_config.py
+++ b/cloudinit/handlers/cloud_config.py
@@ -85,17 +85,16 @@ class CloudConfigPartHandler(handlers.Handler):
         all_mergers.extend(mergers_header)
         if not all_mergers:
             all_mergers = DEF_MERGERS
-        return all_mergers
+        return (payload_yaml, all_mergers)
 
     def _merge_part(self, payload, headers):
-        my_mergers = self._extract_mergers(payload, headers)
+        (payload_yaml, my_mergers) = self._extract_mergers(payload, headers)
         LOG.debug("Merging by applying %s", my_mergers)
         merger = mergers.construct(my_mergers)
         if self.cloud_buf is None:
             # First time through, merge with an empty dict...
             self.cloud_buf = {}
-        self.cloud_buf = merger.merge(self.cloud_buf,
-                                      util.load_yaml(payload))
+        self.cloud_buf = merger.merge(self.cloud_buf, payload_yaml)
 
     def _reset(self):
         self.file_names = []
diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py
index 2702496b..221e93b5 100644
--- a/cloudinit/mergers/__init__.py
+++ b/cloudinit/mergers/__init__.py
@@ -55,6 +55,9 @@ class UnknownMerger(object):
         if not meth:
             meth = self._handle_unknown
             args.insert(0, method_name)
+        LOG.debug("Merging '%s' into '%s' using method '%s' of '%s'",
+                  type_name, type_utils.obj_name(merge_with),
+                  meth.__name__, self)
         return meth(*args)
 
 
@@ -66,6 +69,9 @@ class LookupMerger(UnknownMerger):
         else:
             self._lookups = lookups
 
+    def __str__(self):
+        return 'LookupMerger: (%s)' % (len(self._lookups))
+
     # For items which can not be merged by the parent this object
     # will lookup in a internally maintained set of objects and
     # find which one of those objects can perform the merge. If
@@ -78,6 +84,8 @@ class LookupMerger(UnknownMerger):
                 # First one that has that method/attr gets to be
                 # the one that will be called
                 meth = getattr(merger, meth_wanted)
+                LOG.debug(("Merging using located merger '%s'"
+                            " since it had method '%s'"), merger, meth_wanted)
                 break
         if not meth:
             return UnknownMerger._handle_unknown(self, meth_wanted,
@@ -87,9 +95,9 @@ class LookupMerger(UnknownMerger):
 
 def dict_extract_mergers(config):
     parsed_mergers = []
-    raw_mergers = config.get('merge_how')
+    raw_mergers = config.pop('merge_how', None)
     if raw_mergers is None:
-        raw_mergers = config.get('merge_type')
+        raw_mergers = config.pop('merge_type', None)
     if raw_mergers is None:
         return parsed_mergers
     if isinstance(raw_mergers, (str, basestring)):
diff --git a/cloudinit/mergers/m_dict.py b/cloudinit/mergers/m_dict.py
index 929d3865..2c1c845f 100644
--- a/cloudinit/mergers/m_dict.py
+++ b/cloudinit/mergers/m_dict.py
@@ -20,30 +20,59 @@
 class Merger(object):
     def __init__(self, merger, opts):
         self._merger = merger
-        self._not_overwrite = 'not_overwrite' in opts
+        # Affects merging behavior...
+        self._method = 'replace'
+        for m in ['replace', 'no_replace']:
+            if m in opts:
+                self._method = m
+                break
+        # Affect how recursive merging is done on other primitives
+        self._recurse_str = 'recurse_str' in opts
+        self._recurse_dict = True
+        self._recurse_array = 'recurse_array' in opts
+        self._allow_delete = 'allow_delete' in opts
+
+    def __str__(self):
+        s = ('DictMerger: (method=%s,recurse_str=%s,'
+             'recurse_dict=%s,recurse_array=%s,allow_delete=%s)')
+        s  = s % (self._method,
+                  self._recurse_str,
+                  self._recurse_dict,
+                  self._recurse_array,
+                  self._allow_delete)
+        return s
+
+    def _do_dict_replace(self, value, merge_with, do_replace=True):
+
+        def merge_same_key(old_v, new_v):
+            if do_replace:
+                return new_v
+            if isinstance(new_v, (list, tuple)) and self._recurse_array:
+                return self._merger.merge(old_v, new_v)
+            if isinstance(new_v, (str, basestring)) and self._recurse_str:
+                return self._merger.merge(old_v, new_v)
+            if isinstance(new_v, (dict)) and self._recurse_dict:
+                return self._merger.merge(old_v, new_v)
+            # Otherwise leave it be...
+            return old_v
 
-    # This merging algorithm will attempt to merge with
-    # another dictionary, on encountering any other type of object
-    # it will not merge with said object, but will instead return
-    # the original value
-    #
-    # On encountering a dictionary, it will create a new dictionary
-    # composed of the original and the one to merge with, if 'overwrite'
-    # is enabled then keys that exist in the original will be overwritten
-    # by keys in the one to merge with (and associated values). Otherwise
-    # if not in overwrite mode the 2 conflicting keys themselves will
-    # be merged.
-    def _on_dict(self, value, merge_with):
-        if not isinstance(merge_with, (dict)):
-            return value
-        merged = dict(value)
         for (k, v) in merge_with.items():
-            if k in merged:
-                if self._not_overwrite:
-                    # Attempt to merge them....
-                    merged[k] = self._merger.merge(merged[k], v)
+            if k in value:
+                if v is None and self._allow_delete:
+                    value.pop(k)
                 else:
-                    merged[k] = v
+                    value[k] = merge_same_key(value[k], v)
             else:
-                merged[k] = v
+                value[k] = v
+        return value
+
+    def _on_dict(self, value, merge_with):
+        if not isinstance(merge_with, (dict)):
+            return value
+        if self._method == 'replace':
+            merged = self._do_dict_replace(dict(value), merge_with)
+        elif self._method == 'no_replace':
+            merged = self._do_dict_replace(dict(value), merge_with, False)
+        else:
+            raise NotImplementedError("Unknown merge type %s" % (self._method))
         return merged
diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py
index 208c5f52..c6a23d85 100644
--- a/cloudinit/mergers/m_list.py
+++ b/cloudinit/mergers/m_list.py
@@ -20,18 +20,55 @@
 class Merger(object):
     def __init__(self, merger, opts):
         self._merger = merger
-        self._extend = 'extend' in opts
+        # Affects merging behavior...
+        self._method = 'replace'
+        for m in ['append', 'prepend', 'replace']:
+            if m in opts:
+                self._method = m
+                break
+        # Affect how recursive merging is done on other primitives
+        self._recurse_str = 'recurse_str' in opts
+        self._recurse_dict = 'recurse_dict' in opts
+        self._recurse_array = 'recurse_array' in opts
+
+    def __str__(self):
+        return 'ListMerger: (m=%s,rs=%s,rd=%s,ra=%s)' % (self._method,
+                                                         self._recurse_str,
+                                                         self._recurse_dict,
+                                                         self._recurse_array)
 
     def _on_tuple(self, value, merge_with):
-        return self._on_list(list(value), merge_with)
+        return tuple(self._on_list(list(value), merge_with))
 
-    # On encountering a list or tuple type this action will be applied
-    # a new list will be returned, if the value to merge with is itself
-    # a list and we have been told to 'extend', then the value here will
-    # be extended with the other list.
     def _on_list(self, value, merge_with):
-        if not self._extend or not isinstance(merge_with, (tuple, list)):
-            return merge_with
-        # Leave the original list alone...
-        value = list(value)
-        return value.extend(merge_with)
+        if (self._method == 'replace' and
+            not isinstance(merge_with, (tuple, list))):
+                return merge_with
+
+        # Ok we now know that what we are merging with is a list or tuple.
+        merged_list = []
+        if self._method == 'prepend':
+            merged_list.extend(merge_with)
+            merged_list.extend(value)
+            return merged_list
+        elif self._method == 'append':
+            merged_list.extend(value)
+            merged_list.extend(merge_with)
+            return merged_list
+
+        def merge_same_index(old_v, new_v):
+            if isinstance(new_v, (list, tuple)) and self._recurse_array:
+                return self._merger.merge(old_v, new_v)
+            if isinstance(new_v, (str, basestring)) and self._recurse_str:
+                return self._merger.merge(old_v, new_v)
+            if isinstance(new_v, (dict)) and self._recurse_dict:
+                return self._merger.merge(old_v, new_v)
+            # Otherwise leave it be...
+            return old_v
+
+        # Ok now we are replacing same indexes
+        merged_list.extend(value)
+        common_len = min(len(merged_list), len(merge_with))
+        for i in xrange(0, common_len):
+            merged_list[i] = merge_same_index(merged_list[i], merge_with[i])
+        return merged_list
-- 
cgit v1.2.3


From e14d64a03c6aa3e567b57f0c0a003ca2185f4493 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Tue, 23 Apr 2013 22:53:53 -0700
Subject: Add a bunch of new merging test files + runner.

---
 tests/data/merge_sources/expected1.yaml |   1 +
 tests/data/merge_sources/expected2.yaml |   3 +
 tests/data/merge_sources/expected3.yaml |   1 +
 tests/data/merge_sources/expected4.yaml |   2 +
 tests/data/merge_sources/source1-1.yaml |   3 +
 tests/data/merge_sources/source1-2.yaml |   5 +
 tests/data/merge_sources/source2-1.yaml |   6 +
 tests/data/merge_sources/source2-2.yaml |   5 +
 tests/data/merge_sources/source3-1.yaml |   4 +
 tests/data/merge_sources/source3-2.yaml |   4 +
 tests/data/merge_sources/source4-1.yaml |   3 +
 tests/data/merge_sources/source4-2.yaml |   6 +
 tests/unittests/test_merging.py         | 194 +++++++++-----------------------
 13 files changed, 97 insertions(+), 140 deletions(-)
 create mode 100644 tests/data/merge_sources/expected1.yaml
 create mode 100644 tests/data/merge_sources/expected2.yaml
 create mode 100644 tests/data/merge_sources/expected3.yaml
 create mode 100644 tests/data/merge_sources/expected4.yaml
 create mode 100644 tests/data/merge_sources/source1-1.yaml
 create mode 100644 tests/data/merge_sources/source1-2.yaml
 create mode 100644 tests/data/merge_sources/source2-1.yaml
 create mode 100644 tests/data/merge_sources/source2-2.yaml
 create mode 100644 tests/data/merge_sources/source3-1.yaml
 create mode 100644 tests/data/merge_sources/source3-2.yaml
 create mode 100644 tests/data/merge_sources/source4-1.yaml
 create mode 100644 tests/data/merge_sources/source4-2.yaml

diff --git a/tests/data/merge_sources/expected1.yaml b/tests/data/merge_sources/expected1.yaml
new file mode 100644
index 00000000..640d282b
--- /dev/null
+++ b/tests/data/merge_sources/expected1.yaml
@@ -0,0 +1 @@
+Blah: ['blah2', 'b']
diff --git a/tests/data/merge_sources/expected2.yaml b/tests/data/merge_sources/expected2.yaml
new file mode 100644
index 00000000..6eccc2cf
--- /dev/null
+++ b/tests/data/merge_sources/expected2.yaml
@@ -0,0 +1,3 @@
+Blah: 3
+Blah2: 2
+Blah3: [1]
diff --git a/tests/data/merge_sources/expected3.yaml b/tests/data/merge_sources/expected3.yaml
new file mode 100644
index 00000000..32d9ad48
--- /dev/null
+++ b/tests/data/merge_sources/expected3.yaml
@@ -0,0 +1 @@
+Blah: [blah2, 'blah1']
diff --git a/tests/data/merge_sources/expected4.yaml b/tests/data/merge_sources/expected4.yaml
new file mode 100644
index 00000000..d88d8f73
--- /dev/null
+++ b/tests/data/merge_sources/expected4.yaml
@@ -0,0 +1,2 @@
+#cloud-config
+Blah: {}
diff --git a/tests/data/merge_sources/source1-1.yaml b/tests/data/merge_sources/source1-1.yaml
new file mode 100644
index 00000000..38e4e5e0
--- /dev/null
+++ b/tests/data/merge_sources/source1-1.yaml
@@ -0,0 +1,3 @@
+#cloud-config
+Blah: ['blah2']
+
diff --git a/tests/data/merge_sources/source1-2.yaml b/tests/data/merge_sources/source1-2.yaml
new file mode 100644
index 00000000..2cd0e0e5
--- /dev/null
+++ b/tests/data/merge_sources/source1-2.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+Blah: ['b']
+
+merge_how: 'dict(recurse_array,no_replace)+list(append)'
diff --git a/tests/data/merge_sources/source2-1.yaml b/tests/data/merge_sources/source2-1.yaml
new file mode 100644
index 00000000..c7a33aaa
--- /dev/null
+++ b/tests/data/merge_sources/source2-1.yaml
@@ -0,0 +1,6 @@
+#cloud-config
+
+
+Blah: 1
+Blah2: 2
+Blah3: 3
diff --git a/tests/data/merge_sources/source2-2.yaml b/tests/data/merge_sources/source2-2.yaml
new file mode 100644
index 00000000..8f2fdc1a
--- /dev/null
+++ b/tests/data/merge_sources/source2-2.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+Blah: 3
+Blah2: 2
+Blah3: [1]
diff --git a/tests/data/merge_sources/source3-1.yaml b/tests/data/merge_sources/source3-1.yaml
new file mode 100644
index 00000000..2303e906
--- /dev/null
+++ b/tests/data/merge_sources/source3-1.yaml
@@ -0,0 +1,4 @@
+#cloud-config
+Blah: ['blah1']
+
+
diff --git a/tests/data/merge_sources/source3-2.yaml b/tests/data/merge_sources/source3-2.yaml
new file mode 100644
index 00000000..dca2ad10
--- /dev/null
+++ b/tests/data/merge_sources/source3-2.yaml
@@ -0,0 +1,4 @@
+#cloud-config
+Blah: ['blah2']
+
+merge_how: 'dict(recurse_array,no_replace)+list(prepend)'
diff --git a/tests/data/merge_sources/source4-1.yaml b/tests/data/merge_sources/source4-1.yaml
new file mode 100644
index 00000000..e5b16872
--- /dev/null
+++ b/tests/data/merge_sources/source4-1.yaml
@@ -0,0 +1,3 @@
+#cloud-config
+Blah:
+  b: 1
diff --git a/tests/data/merge_sources/source4-2.yaml b/tests/data/merge_sources/source4-2.yaml
new file mode 100644
index 00000000..1844e0f8
--- /dev/null
+++ b/tests/data/merge_sources/source4-2.yaml
@@ -0,0 +1,6 @@
+#cloud-config
+Blah:
+  b: null
+
+
+merge_how: 'dict(allow_delete,no_replace)+list()'
diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index ad137e85..470b18c7 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -1,142 +1,56 @@
 from tests.unittests import helpers
 
-from cloudinit import mergers
-
-
-class TestSimpleRun(helpers.MockerTestCase):
-    def test_basic_merge(self):
-        source = {
-            'Blah': ['blah2'],
-            'Blah3': 'c',
-        }
-        merge_with = {
-            'Blah2': ['blah3'],
-            'Blah3': 'b',
-            'Blah': ['123'],
-        }
-        # Basic merge should not do thing special
-        merge_how = "list()+dict()+str()"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['Blah'], ['blah2'])
-        self.assertEquals(merged['Blah2'], ['blah3'])
-        self.assertEquals(merged['Blah3'], 'c')
-
-    def test_dict_overwrite(self):
-        source = {
-            'Blah': ['blah2'],
-        }
-        merge_with = {
-            'Blah': ['123'],
-        }
-        # Now lets try a dict overwrite
-        merge_how = "list()+dict(overwrite)+str()"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['Blah'], ['123'])
-
-    def test_string_append(self):
-        source = {
-            'Blah': 'blah2',
-        }
-        merge_with = {
-            'Blah': '345',
-        }
-        merge_how = "list()+dict()+str(append)"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['Blah'], 'blah2345')
-
-    def test_list_extend(self):
-        source = ['abc']
-        merge_with = ['123']
-        merge_how = "list(extend)+dict()+str()"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged, ['abc', '123'])
-
-    def test_deep_merge(self):
-        source = {
-            'a': [1, 'b', 2],
-            'b': 'blahblah',
-            'c': {
-                'e': [1, 2, 3],
-                'f': 'bigblobof',
-                'iamadict': {
-                    'ok': 'ok',
-                }
-            },
-            'run': [
-                'runme',
-                'runme2',
-            ],
-            'runmereally': [
-                'e', ['a'], 'd',
-            ],
-        }
-        merge_with = {
-            'a': ['e', 'f', 'g'],
-            'b': 'more',
-            'c': {
-                'a': 'b',
-                'f': 'stuff',
-            },
-            'run': [
-                'morecmd',
-                'moremoremore',
-            ],
-            'runmereally': [
-                'blah', ['b'], 'e',
-            ],
-        }
-        merge_how = "list(extend)+dict()+str(append)"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['a'], [1, 'b', 2, 'e', 'f', 'g'])
-        self.assertEquals(merged['b'], 'blahblahmore')
-        self.assertEquals(merged['c']['f'], 'bigblobofstuff')
-        self.assertEquals(merged['run'], ['runme', 'runme2', 'morecmd',
-                                          'moremoremore'])
-        self.assertEquals(merged['runmereally'], ['e', ['a'], 'd', 'blah',
-                                                  ['b'], 'e'])
-
-    def test_dict_overwrite_layered(self):
-        source = {
-            'Blah3': {
-                'f': '3',
-                'g': {
-                    'a': 'b',
-                }
-            }
-        }
-        merge_with = {
-            'Blah3': {
-                'e': '2',
-                'g': {
-                    'e': 'f',
-                }
-            }
-        }
-        merge_how = "list()+dict()+str()"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['Blah3'], {
-                'e': '2',
-                'f': '3',
-                'g': {
-                    'a': 'b',
-                    'e': 'f',
-                }
-        })
+from cloudinit.handlers import cloud_config
+from cloudinit.handlers import (CONTENT_START, CONTENT_END)
+
+from cloudinit import helpers as c_helpers
+from cloudinit import util
+
+import collections
+import glob
+import os
+import re
+
+
+class TestSimpleRun(helpers.ResourceUsingTestCase):
+    def _load_merge_files(self, data_dir):
+        merge_root = self.resourceLocation(data_dir)
+        tests = []
+        source_ids = collections.defaultdict(list)
+        expected_files = {}
+        for fn in glob.glob(os.path.join(merge_root, "source*.*yaml")):
+            base_fn = os.path.basename(fn)
+            file_id = re.match(r"source(\d+)\-(\d+)[.]yaml", base_fn)
+            if not file_id:
+                raise IOError("File %s does not have a numeric identifier"
+                              % (fn))
+            file_id = int(file_id.group(1))
+            source_ids[file_id].append(fn)
+            expected_fn = os.path.join(merge_root,
+                                       "expected%s.yaml" % (file_id))
+            if not os.path.isfile(expected_fn):
+                raise IOError("No expected file found at %s" % (expected_fn))
+            expected_files[file_id] = expected_fn
+        for id in sorted(source_ids.keys()):
+            source_file_contents = []
+            for fn in sorted(source_ids[id]):
+                source_file_contents.append(util.load_file(fn))
+            expected = util.load_yaml(util.load_file(expected_files[id]))
+            tests.append((source_file_contents, expected))
+        return tests
+
+    def test_merge_samples(self):
+        tests = self._load_merge_files('merge_sources')
+        paths = c_helpers.Paths({})
+        cc_handler = cloud_config.CloudConfigPartHandler(paths)
+        cc_handler.cloud_fn = None
+        for (payloads, expected_merge) in tests:
+            cc_handler.handle_part(None, CONTENT_START, None,
+                                   None, None, None)
+            for (i, p) in enumerate(payloads):
+                cc_handler.handle_part(None, None, "t-%s.yaml" % (i + 1),
+                                       p, None, {})
+            merged_buf = cc_handler.cloud_buf
+            cc_handler.handle_part(None, CONTENT_END, None,
+                                   None, None, None)
+            self.assertEquals(expected_merge, merged_buf)
-- 
cgit v1.2.3


From 4a669649b17cf01b6f89f7902b6683d02ef0bee1 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Fri, 3 May 2013 14:41:28 -0700
Subject: More merging adjustments.

Looks like this should be in pretty good shape and has passed some
of the basic backwards compat. merging tests that I added.
---
 cloudinit/mergers/__init__.py           |   2 +-
 cloudinit/mergers/m_dict.py             |  27 ++++---
 cloudinit/mergers/m_list.py             |   9 +--
 cloudinit/mergers/m_str.py              |  44 ++++++++++++
 tests/data/merge_sources/expected2.yaml |   4 +-
 tests/data/merge_sources/expected5.yaml |   7 ++
 tests/data/merge_sources/source5-1.yaml |   6 ++
 tests/data/merge_sources/source5-2.yaml |   8 +++
 tests/unittests/test_merging.py         | 123 +++++++++++++++++++++++++++-----
 tests/unittests/test_userdata.py        |   5 +-
 10 files changed, 201 insertions(+), 34 deletions(-)
 create mode 100644 cloudinit/mergers/m_str.py
 create mode 100644 tests/data/merge_sources/expected5.yaml
 create mode 100644 tests/data/merge_sources/source5-1.yaml
 create mode 100644 tests/data/merge_sources/source5-2.yaml

diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py
index 221e93b5..0978b2c6 100644
--- a/cloudinit/mergers/__init__.py
+++ b/cloudinit/mergers/__init__.py
@@ -25,7 +25,7 @@ from cloudinit import type_utils
 NAME_MTCH = re.compile(r"(^[a-zA-Z_][A-Za-z0-9_]*)\((.*?)\)$")
 
 LOG = logging.getLogger(__name__)
-DEF_MERGE_TYPE = "list()+dict()"
+DEF_MERGE_TYPE = "list()+dict()+str()"
 MERGER_PREFIX = 'm_'
 MERGER_ATTR = 'Merger'
 
diff --git a/cloudinit/mergers/m_dict.py b/cloudinit/mergers/m_dict.py
index 2c1c845f..82caa004 100644
--- a/cloudinit/mergers/m_dict.py
+++ b/cloudinit/mergers/m_dict.py
@@ -16,21 +16,32 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+DEF_MERGE_TYPE = 'no_replace'
+MERGE_TYPES = ('replace', DEF_MERGE_TYPE,)
+
+
+def _has_any(what, *keys):
+    for k in keys:
+        if k in what:
+            return True
+    return False
+
 
 class Merger(object):
     def __init__(self, merger, opts):
         self._merger = merger
         # Affects merging behavior...
-        self._method = 'replace'
-        for m in ['replace', 'no_replace']:
+        self._method = DEF_MERGE_TYPE
+        for m in MERGE_TYPES:
             if m in opts:
                 self._method = m
                 break
-        # Affect how recursive merging is done on other primitives
+        # Affect how recursive merging is done on other primitives.
         self._recurse_str = 'recurse_str' in opts
-        self._recurse_dict = True
-        self._recurse_array = 'recurse_array' in opts
+        self._recurse_array = _has_any(opts, 'recurse_array', 'recurse_list')
         self._allow_delete = 'allow_delete' in opts
+        # Backwards compat require this to be on.
+        self._recurse_dict = True
 
     def __str__(self):
         s = ('DictMerger: (method=%s,recurse_str=%s,'
@@ -42,14 +53,14 @@ class Merger(object):
                   self._allow_delete)
         return s
 
-    def _do_dict_replace(self, value, merge_with, do_replace=True):
+    def _do_dict_replace(self, value, merge_with, do_replace):
 
         def merge_same_key(old_v, new_v):
             if do_replace:
                 return new_v
             if isinstance(new_v, (list, tuple)) and self._recurse_array:
                 return self._merger.merge(old_v, new_v)
-            if isinstance(new_v, (str, basestring)) and self._recurse_str:
+            if isinstance(new_v, (basestring)) and self._recurse_str:
                 return self._merger.merge(old_v, new_v)
             if isinstance(new_v, (dict)) and self._recurse_dict:
                 return self._merger.merge(old_v, new_v)
@@ -70,7 +81,7 @@ class Merger(object):
         if not isinstance(merge_with, (dict)):
             return value
         if self._method == 'replace':
-            merged = self._do_dict_replace(dict(value), merge_with)
+            merged = self._do_dict_replace(dict(value), merge_with, True)
         elif self._method == 'no_replace':
             merged = self._do_dict_replace(dict(value), merge_with, False)
         else:
diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py
index c6a23d85..50f279e8 100644
--- a/cloudinit/mergers/m_list.py
+++ b/cloudinit/mergers/m_list.py
@@ -32,10 +32,11 @@ class Merger(object):
         self._recurse_array = 'recurse_array' in opts
 
     def __str__(self):
-        return 'ListMerger: (m=%s,rs=%s,rd=%s,ra=%s)' % (self._method,
-                                                         self._recurse_str,
-                                                         self._recurse_dict,
-                                                         self._recurse_array)
+        return ('ListMerger: (method=%s,recurse_str=%s,'
+                'recurse_dict=%s,recurse_array=%s)') % (self._method,
+                                                        self._recurse_str,
+                                                        self._recurse_dict,
+                                                        self._recurse_array)
 
     def _on_tuple(self, value, merge_with):
         return tuple(self._on_list(list(value), merge_with))
diff --git a/cloudinit/mergers/m_str.py b/cloudinit/mergers/m_str.py
new file mode 100644
index 00000000..e22ce28a
--- /dev/null
+++ b/cloudinit/mergers/m_str.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+class Merger(object):
+    def __init__(self, _merger, opts):
+        self._append = 'append' in opts
+
+    def __str__(self):
+        return 'StringMerger: (append=%s)' % (self._append)
+
+    # On encountering a unicode object to merge value with
+    # we will for now just proxy into the string method to let it handle it.
+    def _on_unicode(self, value, merge_with):
+        return self._on_str(value, merge_with)
+
+    # On encountering a string object to merge with we will
+    # perform the following action, if appending we will
+    # merge them together, otherwise we will just return value.
+    def _on_str(self, value, merge_with):
+        if not isinstance(value, (basestring)):
+            return merge_with
+        if not self._append:
+            return merge_with
+        if isinstance(value, unicode):
+            return value + unicode(merge_with)
+        else:
+            return value + str(merge_with)
diff --git a/tests/data/merge_sources/expected2.yaml b/tests/data/merge_sources/expected2.yaml
index 6eccc2cf..f5312eb1 100644
--- a/tests/data/merge_sources/expected2.yaml
+++ b/tests/data/merge_sources/expected2.yaml
@@ -1,3 +1,3 @@
-Blah: 3
+Blah: 1
 Blah2: 2
-Blah3: [1]
+Blah3: 3
diff --git a/tests/data/merge_sources/expected5.yaml b/tests/data/merge_sources/expected5.yaml
new file mode 100644
index 00000000..628f5878
--- /dev/null
+++ b/tests/data/merge_sources/expected5.yaml
@@ -0,0 +1,7 @@
+#cloud-config
+
+Blah: 3
+Blah2: 2
+Blah3: [1]
+
+
diff --git a/tests/data/merge_sources/source5-1.yaml b/tests/data/merge_sources/source5-1.yaml
new file mode 100644
index 00000000..c7a33aaa
--- /dev/null
+++ b/tests/data/merge_sources/source5-1.yaml
@@ -0,0 +1,6 @@
+#cloud-config
+
+
+Blah: 1
+Blah2: 2
+Blah3: 3
diff --git a/tests/data/merge_sources/source5-2.yaml b/tests/data/merge_sources/source5-2.yaml
new file mode 100644
index 00000000..f61c96a2
--- /dev/null
+++ b/tests/data/merge_sources/source5-2.yaml
@@ -0,0 +1,8 @@
+#cloud-config
+
+Blah: 3
+Blah2: 2
+Blah3: [1]
+
+
+merge_how: 'dict(replace)+list(append)'
diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index 470b18c7..f83522d7 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -11,14 +11,39 @@ import glob
 import os
 import re
 
+SOURCE_PAT = "source*.*yaml"
+EXPECTED_PAT = "expected%s.yaml" 
+
+
+def _old_mergedict(src, cand):
+    """
+    Merge values from C{cand} into C{src}.
+    If C{src} has a key C{cand} will not override.
+    Nested dictionaries are merged recursively.
+    """
+    if isinstance(src, dict) and isinstance(cand, dict):
+        for (k, v) in cand.iteritems():
+            if k not in src:
+                src[k] = v
+            else:
+                src[k] = _old_mergedict(src[k], v)
+    return src
+
+
+def _old_mergemanydict(*args):
+    out = {}
+    for a in args:
+        out = _old_mergedict(out, a)
+    return out
+
 
 class TestSimpleRun(helpers.ResourceUsingTestCase):
-    def _load_merge_files(self, data_dir):
-        merge_root = self.resourceLocation(data_dir)
+    def _load_merge_files(self):
+        merge_root = self.resourceLocation('merge_sources')
         tests = []
         source_ids = collections.defaultdict(list)
         expected_files = {}
-        for fn in glob.glob(os.path.join(merge_root, "source*.*yaml")):
+        for fn in glob.glob(os.path.join(merge_root, SOURCE_PAT)):
             base_fn = os.path.basename(fn)
             file_id = re.match(r"source(\d+)\-(\d+)[.]yaml", base_fn)
             if not file_id:
@@ -26,31 +51,97 @@ class TestSimpleRun(helpers.ResourceUsingTestCase):
                               % (fn))
             file_id = int(file_id.group(1))
             source_ids[file_id].append(fn)
-            expected_fn = os.path.join(merge_root,
-                                       "expected%s.yaml" % (file_id))
+            expected_fn = os.path.join(merge_root, EXPECTED_PAT % (file_id))
             if not os.path.isfile(expected_fn):
                 raise IOError("No expected file found at %s" % (expected_fn))
             expected_files[file_id] = expected_fn
-        for id in sorted(source_ids.keys()):
+        for i in sorted(source_ids.keys()):
             source_file_contents = []
-            for fn in sorted(source_ids[id]):
-                source_file_contents.append(util.load_file(fn))
-            expected = util.load_yaml(util.load_file(expected_files[id]))
-            tests.append((source_file_contents, expected))
+            for fn in sorted(source_ids[i]):
+                source_file_contents.append([fn, util.load_file(fn)])
+            expected = util.load_yaml(util.load_file(expected_files[i]))
+            entry = [source_file_contents, [expected, expected_files[i]]]
+            tests.append(entry)
         return tests
 
     def test_merge_samples(self):
-        tests = self._load_merge_files('merge_sources')
+        tests = self._load_merge_files()
         paths = c_helpers.Paths({})
         cc_handler = cloud_config.CloudConfigPartHandler(paths)
         cc_handler.cloud_fn = None
-        for (payloads, expected_merge) in tests:
+        for (payloads, (expected_merge, expected_fn)) in tests:
             cc_handler.handle_part(None, CONTENT_START, None,
                                    None, None, None)
-            for (i, p) in enumerate(payloads):
-                cc_handler.handle_part(None, None, "t-%s.yaml" % (i + 1),
-                                       p, None, {})
+            merging_fns = []
+            for (fn, contents) in payloads:
+                cc_handler.handle_part(None, None, "%s.yaml" % (fn),
+                                       contents, None, {})
+                merging_fns.append(fn)
             merged_buf = cc_handler.cloud_buf
             cc_handler.handle_part(None, CONTENT_END, None,
                                    None, None, None)
-            self.assertEquals(expected_merge, merged_buf)
+            fail_msg = "Equality failure on checking %s with %s: %s != %s"
+            fail_msg = fail_msg % (expected_fn,
+                                   ",".join(merging_fns), merged_buf,
+                                   expected_merge)
+            self.assertEquals(expected_merge, merged_buf, msg=fail_msg)
+
+    def test_compat_merges_dict(self):
+        a = {
+            '1': '2',
+            'b': 'c',
+        }
+        b = {
+            'b': 'e',
+        }
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)
+
+    def test_compat_merges_list(self):
+        a = {'b': [1, 2, 3]}
+        b = {'b': [4, 5]}
+        c = {'b': [6, 7]}
+        e = _old_mergemanydict(a, b, c)
+        f = util.mergemanydict([a, b, c])
+        self.assertEquals(e, f)
+
+    def test_compat_merges_str(self):
+        a = {'b': "hi"}
+        b = {'b': "howdy"}
+        c = {'b': "hallo"}
+        e = _old_mergemanydict(a, b, c)
+        f = util.mergemanydict([a, b, c])
+        self.assertEquals(e, f)
+
+    def test_compat_merge_sub_dict(self):
+        a = {
+            '1': '2',
+            'b': {
+                'f': 'g',
+            }
+        }
+        b = {
+            'b': {
+                'e': 'c',
+            }
+        }
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)
+
+    def test_compat_merge_sub_list(self):
+        a = {
+            '1': '2',
+            'b': {
+                'f': ['1'],
+            }
+        }
+        b = {
+            'b': {
+                'f': [],
+            }
+        }
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)
diff --git a/tests/unittests/test_userdata.py b/tests/unittests/test_userdata.py
index fdfe2542..5fb9acd9 100644
--- a/tests/unittests/test_userdata.py
+++ b/tests/unittests/test_userdata.py
@@ -60,7 +60,6 @@ run:
  - c
 '''
         message1 = MIMEBase("text", "cloud-config")
-        message1['Merge-Type'] = 'dict()+list(extend)+str(append)'
         message1.set_payload(blob)
 
         blob2 = '''
@@ -72,7 +71,8 @@ run:
  - morestuff
 '''
         message2 = MIMEBase("text", "cloud-config")
-        message2['X-Merge-Type'] = 'dict()+list(extend)+str()'
+        message2['X-Merge-Type'] = ('dict(recurse_array,'
+                                    'recurse_str)+list(append)+str(append)')
         message2.set_payload(blob2)
 
         blob3 = '''
@@ -84,7 +84,6 @@ e:
 p: 1
 '''
         message3 = MIMEBase("text", "cloud-config")
-        message3['Merge-Type'] = 'dict()+list()+str()'
         message3.set_payload(blob3)
 
         messages = [message1, message2, message3]
-- 
cgit v1.2.3


From c972396ecccb4b67eafc038a482ffeaa1df2c93e Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Fri, 3 May 2013 14:46:52 -0700
Subject: A few more tests + cleanings.

---
 cloudinit/mergers/m_list.py     |  7 +++++--
 tests/unittests/test_merging.py | 24 ++++++++++++++++++++++++
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py
index 50f279e8..34b32379 100644
--- a/cloudinit/mergers/m_list.py
+++ b/cloudinit/mergers/m_list.py
@@ -16,13 +16,16 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+DEF_MERGE_TYPE = 'replace'
+MERGE_TYPES = ('append', 'prepend', DEF_MERGE_TYPE,)
+
 
 class Merger(object):
     def __init__(self, merger, opts):
         self._merger = merger
         # Affects merging behavior...
-        self._method = 'replace'
-        for m in ['append', 'prepend', 'replace']:
+        self._method = DEF_MERGE_TYPE
+        for m in MERGE_TYPES:
             if m in opts:
                 self._method = m
                 break
diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index f83522d7..cff8ac12 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -115,6 +115,30 @@ class TestSimpleRun(helpers.ResourceUsingTestCase):
         self.assertEquals(e, f)
 
     def test_compat_merge_sub_dict(self):
+        a = {
+            '1': '2',
+            'b': {
+                'f': 'g',
+                'e': 'c',
+                'h': 'd',
+                'hh': {
+                    '1': 2,
+                },
+            }
+        }
+        b = {
+            'b': {
+                'e': 'c',
+                'hh': {
+                    '3': 4,
+                }
+            }
+        }
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)
+
+    def test_compat_merge_sub_dict2(self):
         a = {
             '1': '2',
             'b': {
-- 
cgit v1.2.3


From 5118a33b22a376954bd048c3142f2d3f7f55d003 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Fri, 3 May 2013 14:49:16 -0700
Subject: Rename the merge cc sample function name.

---
 tests/unittests/test_merging.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index cff8ac12..dd8c2eee 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -64,7 +64,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase):
             tests.append(entry)
         return tests
 
-    def test_merge_samples(self):
+    def test_merge_cc_samples(self):
         tests = self._load_merge_files()
         paths = c_helpers.Paths({})
         cc_handler = cloud_config.CloudConfigPartHandler(paths)
-- 
cgit v1.2.3


From 9a1584b701cecbbba4a9371542114bcc806ec596 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Fri, 3 May 2013 15:05:45 -0700
Subject: A few pep8/pylint cleanups.

---
 cloudinit/mergers/m_dict.py     | 7 ++-----
 cloudinit/mergers/m_list.py     | 2 +-
 tests/unittests/test_merging.py | 2 +-
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/cloudinit/mergers/m_dict.py b/cloudinit/mergers/m_dict.py
index 82caa004..a16141fa 100644
--- a/cloudinit/mergers/m_dict.py
+++ b/cloudinit/mergers/m_dict.py
@@ -46,11 +46,8 @@ class Merger(object):
     def __str__(self):
         s = ('DictMerger: (method=%s,recurse_str=%s,'
              'recurse_dict=%s,recurse_array=%s,allow_delete=%s)')
-        s  = s % (self._method,
-                  self._recurse_str,
-                  self._recurse_dict,
-                  self._recurse_array,
-                  self._allow_delete)
+        s = s % (self._method, self._recurse_str,
+                 self._recurse_dict, self._recurse_array, self._allow_delete)
         return s
 
     def _do_dict_replace(self, value, merge_with, do_replace):
diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py
index 34b32379..8a0b5827 100644
--- a/cloudinit/mergers/m_list.py
+++ b/cloudinit/mergers/m_list.py
@@ -47,7 +47,7 @@ class Merger(object):
     def _on_list(self, value, merge_with):
         if (self._method == 'replace' and
             not isinstance(merge_with, (tuple, list))):
-                return merge_with
+            return merge_with
 
         # Ok we now know that what we are merging with is a list or tuple.
         merged_list = []
diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index dd8c2eee..ac2ccad4 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -12,7 +12,7 @@ import os
 import re
 
 SOURCE_PAT = "source*.*yaml"
-EXPECTED_PAT = "expected%s.yaml" 
+EXPECTED_PAT = "expected%s.yaml"
 
 
 def _old_mergedict(src, cand):
-- 
cgit v1.2.3


From 563af0754dc53fe4a95a4dee8ed18282f7a38104 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Sat, 4 May 2013 12:57:15 -0700
Subject: Add a set of randomized (seeded) dict tests.

---
 tests/unittests/test_merging.py | 69 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index ac2ccad4..dddf8c6c 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -9,10 +9,13 @@ from cloudinit import util
 import collections
 import glob
 import os
+import random
 import re
+import string
 
 SOURCE_PAT = "source*.*yaml"
 EXPECTED_PAT = "expected%s.yaml"
+TYPES = [long, int, dict, str, list, tuple, None]
 
 
 def _old_mergedict(src, cand):
@@ -37,6 +40,60 @@ def _old_mergemanydict(*args):
     return out
 
 
+def _random_str(rand):
+    base = ''
+    for _i in xrange(rand.randint(1, 2**8)):
+        base += rand.choice(string.letters + string.digits)
+    return base
+
+
+class _NoMoreException(Exception):
+    pass
+
+
+def _make_dict(current_depth, max_depth, rand):
+    if current_depth >= max_depth:
+        raise _NoMoreException()
+    if current_depth == 0:
+        t = dict
+    else:
+        t = rand.choice(TYPES)
+    base = None
+    if t in [None]:
+        return base
+    if t in [dict, list, tuple]:
+        if t in [dict]:
+            amount = rand.randint(0, 5)
+            keys = [_random_str(rand) for _i in xrange(0, amount)]
+            base = {}
+            for k in keys:
+                try:
+                    base[k] = _make_dict(current_depth + 1, max_depth, rand)
+                except _NoMoreException:
+                    pass
+        elif t in [list, tuple]:
+            base = []
+            amount = rand.randint(0, 5)
+            for _i in xrange(0, amount):
+                try:
+                    base.append(_make_dict(current_depth + 1, max_depth, rand))
+                except _NoMoreException:
+                    pass
+            if t in [tuple]:
+                base = tuple(base)
+    elif t in [long, int]:
+        base = rand.randint(0, 2**8)
+    elif t in [str]:
+        base = _random_str(rand)
+    return base
+
+
+def make_dict(max_depth, seed=None):
+    max_depth = max(1, max_depth)
+    rand = random.Random(seed)
+    return _make_dict(0, max_depth, rand)
+
+
 class TestSimpleRun(helpers.ResourceUsingTestCase):
     def _load_merge_files(self):
         merge_root = self.resourceLocation('merge_sources')
@@ -64,6 +121,18 @@ class TestSimpleRun(helpers.ResourceUsingTestCase):
             tests.append(entry)
         return tests
 
+    def test_seed_runs(self):
+        test_dicts = []
+        for i in range(1, 50):
+            base_dicts = []
+            for j in range(1, 50):
+                base_dicts.append(make_dict(5, i * j))
+            test_dicts.append(base_dicts)
+        for test in test_dicts:
+            c = _old_mergemanydict(*test)
+            d = util.mergemanydict(test)
+            self.assertEquals(c, d)
+
     def test_merge_cc_samples(self):
         tests = self._load_merge_files()
         paths = c_helpers.Paths({})
-- 
cgit v1.2.3


From 229df67191e7dff058151f1e1f6e007667d55d9c Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Tue, 7 May 2013 21:38:06 -0700
Subject: A few more test files.

---
 cloudinit/mergers/m_list.py             | 12 ++++++++---
 tests/data/merge_sources/expected6.yaml |  9 ++++++++
 tests/data/merge_sources/expected7.yaml | 38 +++++++++++++++++++++++++++++++++
 tests/data/merge_sources/expected8.yaml |  7 ++++++
 tests/data/merge_sources/expected9.yaml |  5 +++++
 tests/data/merge_sources/source6-1.yaml |  5 +++++
 tests/data/merge_sources/source6-2.yaml |  8 +++++++
 tests/data/merge_sources/source7-1.yaml | 27 +++++++++++++++++++++++
 tests/data/merge_sources/source7-2.yaml | 17 +++++++++++++++
 tests/data/merge_sources/source8-1.yaml |  7 ++++++
 tests/data/merge_sources/source8-2.yaml |  6 ++++++
 tests/data/merge_sources/source9-1.yaml |  5 +++++
 tests/data/merge_sources/source9-2.yaml |  6 ++++++
 13 files changed, 149 insertions(+), 3 deletions(-)
 create mode 100644 tests/data/merge_sources/expected6.yaml
 create mode 100644 tests/data/merge_sources/expected7.yaml
 create mode 100644 tests/data/merge_sources/expected8.yaml
 create mode 100644 tests/data/merge_sources/expected9.yaml
 create mode 100644 tests/data/merge_sources/source6-1.yaml
 create mode 100644 tests/data/merge_sources/source6-2.yaml
 create mode 100644 tests/data/merge_sources/source7-1.yaml
 create mode 100644 tests/data/merge_sources/source7-2.yaml
 create mode 100644 tests/data/merge_sources/source8-1.yaml
 create mode 100644 tests/data/merge_sources/source8-2.yaml
 create mode 100644 tests/data/merge_sources/source9-1.yaml
 create mode 100644 tests/data/merge_sources/source9-2.yaml

diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py
index 8a0b5827..1184ded7 100644
--- a/cloudinit/mergers/m_list.py
+++ b/cloudinit/mergers/m_list.py
@@ -19,6 +19,12 @@
 DEF_MERGE_TYPE = 'replace'
 MERGE_TYPES = ('append', 'prepend', DEF_MERGE_TYPE,)
 
+def _has_any(what, *keys):
+    for k in keys:
+        if k in what:
+            return True
+    return False
+
 
 class Merger(object):
     def __init__(self, merger, opts):
@@ -30,9 +36,9 @@ class Merger(object):
                 self._method = m
                 break
         # Affect how recursive merging is done on other primitives
-        self._recurse_str = 'recurse_str' in opts
-        self._recurse_dict = 'recurse_dict' in opts
-        self._recurse_array = 'recurse_array' in opts
+        self._recurse_str = _has_any(opts, 'recurse_str')
+        self._recurse_dict = _has_any(opts, 'recurse_dict')
+        self._recurse_array = _has_any(opts, 'recurse_array', 'recurse_list')
 
     def __str__(self):
         return ('ListMerger: (method=%s,recurse_str=%s,'
diff --git a/tests/data/merge_sources/expected6.yaml b/tests/data/merge_sources/expected6.yaml
new file mode 100644
index 00000000..7afe1d7c
--- /dev/null
+++ b/tests/data/merge_sources/expected6.yaml
@@ -0,0 +1,9 @@
+#cloud-config
+
+run_cmds:
+   - bash
+   - top
+   - ps
+   - vi
+   - emacs
+
diff --git a/tests/data/merge_sources/expected7.yaml b/tests/data/merge_sources/expected7.yaml
new file mode 100644
index 00000000..25284f04
--- /dev/null
+++ b/tests/data/merge_sources/expected7.yaml
@@ -0,0 +1,38 @@
+#cloud-config
+
+users:
+  - default
+  - name: foobar
+    gecos: Foo B. Bar
+    primary-group: foobar
+    groups: users
+    selinux-user: staff_u
+    expiredate: 2012-09-01
+    ssh-import-id: foobar
+    lock-passwd: false
+    passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
+  - name: barfoo
+    gecos: Bar B. Foo
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    groups: users, admin
+    ssh-import-id: None
+    lock-passwd: true
+    ssh-authorized-keys:
+      - <ssh pub key 1>
+      - <ssh pub key 2>
+  - name: cloudy
+    gecos: Magic Cloud App Daemon User
+    inactive: true
+    system: true
+  - bob
+  - joe
+  - sue
+  - name: foobar_jr
+    gecos: Foo B. Bar Jr
+    primary-group: foobar
+    groups: users
+    selinux-user: staff_u
+    expiredate: 2012-09-01
+    ssh-import-id: foobar
+    lock-passwd: false
+    passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
diff --git a/tests/data/merge_sources/expected8.yaml b/tests/data/merge_sources/expected8.yaml
new file mode 100644
index 00000000..69ca562d
--- /dev/null
+++ b/tests/data/merge_sources/expected8.yaml
@@ -0,0 +1,7 @@
+#cloud-config
+
+mounts:
+ - [ ephemeral22, /mnt, auto, "defaults,noexec" ]
+ - [ sdc, /opt/data ]
+ - [ xvdh, /opt/data, "auto", "defaults,nobootwait", "0", "0" ]
+ - [ dd, /dev/zero ]
diff --git a/tests/data/merge_sources/expected9.yaml b/tests/data/merge_sources/expected9.yaml
new file mode 100644
index 00000000..00f91ca0
--- /dev/null
+++ b/tests/data/merge_sources/expected9.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+phone_home:
+ url: http://my.example.com/$INSTANCE_ID/$BLAH_BLAH
+ post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id ]
diff --git a/tests/data/merge_sources/source6-1.yaml b/tests/data/merge_sources/source6-1.yaml
new file mode 100644
index 00000000..519f7309
--- /dev/null
+++ b/tests/data/merge_sources/source6-1.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+run_cmds:
+  - bash
+  - top
diff --git a/tests/data/merge_sources/source6-2.yaml b/tests/data/merge_sources/source6-2.yaml
new file mode 100644
index 00000000..d8fac446
--- /dev/null
+++ b/tests/data/merge_sources/source6-2.yaml
@@ -0,0 +1,8 @@
+#cloud-config
+
+run_cmds:
+  - ps
+  - vi
+  - emacs
+
+merge_type: 'list(append)+dict(recurse_array)+str()'
diff --git a/tests/data/merge_sources/source7-1.yaml b/tests/data/merge_sources/source7-1.yaml
new file mode 100644
index 00000000..8fb9b32a
--- /dev/null
+++ b/tests/data/merge_sources/source7-1.yaml
@@ -0,0 +1,27 @@
+#cloud-config
+
+users:
+  - default
+  - name: foobar
+    gecos: Foo B. Bar
+    primary-group: foobar
+    groups: users
+    selinux-user: staff_u
+    expiredate: 2012-09-01
+    ssh-import-id: foobar
+    lock-passwd: false
+    passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
+  - name: barfoo
+    gecos: Bar B. Foo
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    groups: users, admin
+    ssh-import-id: None
+    lock-passwd: true
+    ssh-authorized-keys:
+      - <ssh pub key 1>
+      - <ssh pub key 2>
+  - name: cloudy
+    gecos: Magic Cloud App Daemon User
+    inactive: true
+    system: true
+
diff --git a/tests/data/merge_sources/source7-2.yaml b/tests/data/merge_sources/source7-2.yaml
new file mode 100644
index 00000000..1e26201b
--- /dev/null
+++ b/tests/data/merge_sources/source7-2.yaml
@@ -0,0 +1,17 @@
+#cloud-config
+
+users:
+  - bob
+  - joe
+  - sue
+  - name: foobar_jr
+    gecos: Foo B. Bar Jr
+    primary-group: foobar
+    groups: users
+    selinux-user: staff_u
+    expiredate: 2012-09-01
+    ssh-import-id: foobar
+    lock-passwd: false
+    passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
+
+merge_how: "dict(recurse_array)+list(append)"
diff --git a/tests/data/merge_sources/source8-1.yaml b/tests/data/merge_sources/source8-1.yaml
new file mode 100644
index 00000000..5ea51c2c
--- /dev/null
+++ b/tests/data/merge_sources/source8-1.yaml
@@ -0,0 +1,7 @@
+#cloud-config
+
+mounts:
+ - [ ephemeral0, /mnt, auto, "defaults,noexec" ]
+ - [ sdc, /opt/data ]
+ - [ xvdh, /opt/data, "auto", "defaults,nobootwait", "0", "0" ]
+ - [ dd, /dev/zero ]
diff --git a/tests/data/merge_sources/source8-2.yaml b/tests/data/merge_sources/source8-2.yaml
new file mode 100644
index 00000000..7fa3262b
--- /dev/null
+++ b/tests/data/merge_sources/source8-2.yaml
@@ -0,0 +1,6 @@
+#cloud-config
+
+mounts:
+ - [ ephemeral22, /mnt, auto, "defaults,noexec" ]
+
+merge_how: 'dict(recurse_array)+list(recurse_list,recurse_str)+str()'
diff --git a/tests/data/merge_sources/source9-1.yaml b/tests/data/merge_sources/source9-1.yaml
new file mode 100644
index 00000000..0b102ba6
--- /dev/null
+++ b/tests/data/merge_sources/source9-1.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+phone_home:
+ url: http://my.example.com/$INSTANCE_ID/
+ post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id ]
diff --git a/tests/data/merge_sources/source9-2.yaml b/tests/data/merge_sources/source9-2.yaml
new file mode 100644
index 00000000..ac85afc6
--- /dev/null
+++ b/tests/data/merge_sources/source9-2.yaml
@@ -0,0 +1,6 @@
+#cloud-config
+
+phone_home:
+ url: $BLAH_BLAH
+
+merge_how: 'dict(recurse_str)+str(append)'
-- 
cgit v1.2.3


From ff232886555964220769da6d8b73198b5d51ef16 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Tue, 7 May 2013 21:42:32 -0700
Subject: 1 more test that does some list appending.

---
 tests/data/merge_sources/expected10.yaml | 7 +++++++
 tests/data/merge_sources/source10-1.yaml | 6 ++++++
 tests/data/merge_sources/source10-2.yaml | 6 ++++++
 3 files changed, 19 insertions(+)
 create mode 100644 tests/data/merge_sources/expected10.yaml
 create mode 100644 tests/data/merge_sources/source10-1.yaml
 create mode 100644 tests/data/merge_sources/source10-2.yaml

diff --git a/tests/data/merge_sources/expected10.yaml b/tests/data/merge_sources/expected10.yaml
new file mode 100644
index 00000000..b865db16
--- /dev/null
+++ b/tests/data/merge_sources/expected10.yaml
@@ -0,0 +1,7 @@
+#cloud-config
+
+power_state:
+  delay: 30
+  mode: poweroff
+  message: [Bye, Bye, Pew, Pew]
+  
diff --git a/tests/data/merge_sources/source10-1.yaml b/tests/data/merge_sources/source10-1.yaml
new file mode 100644
index 00000000..6ae72a13
--- /dev/null
+++ b/tests/data/merge_sources/source10-1.yaml
@@ -0,0 +1,6 @@
+#cloud-config
+
+power_state:
+ delay: 30
+ mode: poweroff
+ message: [Bye, Bye]
diff --git a/tests/data/merge_sources/source10-2.yaml b/tests/data/merge_sources/source10-2.yaml
new file mode 100644
index 00000000..a38cf1c5
--- /dev/null
+++ b/tests/data/merge_sources/source10-2.yaml
@@ -0,0 +1,6 @@
+#cloud-config
+
+power_state:
+  message: [Pew, Pew]
+  
+merge_how: 'dict(recurse_list)+list(append)'
-- 
cgit v1.2.3


From 2b351c5435939d16ba06ec0c45847d47f4b21d51 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Thu, 9 May 2013 22:34:31 -0700
Subject: Fix the cloud config merging so that it is backwards compat.

The new change for merging works well in the mergedict case
but the default merging type for cloud config needs to reflect
how yaml was loaded in bulk, which is the same as the replacing
keys merging type that is now provided.
---
 cloudinit/handlers/cloud_config.py       | 15 +++++++++++++--
 tests/data/merge_sources/expected11.yaml |  5 +++++
 tests/data/merge_sources/expected12.yaml |  5 +++++
 tests/data/merge_sources/expected2.yaml  |  4 ++--
 tests/data/merge_sources/source11-1.yaml |  5 +++++
 tests/data/merge_sources/source11-2.yaml |  3 +++
 tests/data/merge_sources/source11-3.yaml |  3 +++
 tests/data/merge_sources/source12-1.yaml |  8 ++++++++
 tests/data/merge_sources/source12-2.yaml |  5 +++++
 tests/unittests/test_merging.py          | 15 +++++++++++++++
 tests/unittests/test_userdata.py         |  2 +-
 11 files changed, 65 insertions(+), 5 deletions(-)
 create mode 100644 tests/data/merge_sources/expected11.yaml
 create mode 100644 tests/data/merge_sources/expected12.yaml
 create mode 100644 tests/data/merge_sources/source11-1.yaml
 create mode 100644 tests/data/merge_sources/source11-2.yaml
 create mode 100644 tests/data/merge_sources/source11-3.yaml
 create mode 100644 tests/data/merge_sources/source12-1.yaml
 create mode 100644 tests/data/merge_sources/source12-2.yaml

diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py
index 2ae9b226..529109ce 100644
--- a/cloudinit/handlers/cloud_config.py
+++ b/cloudinit/handlers/cloud_config.py
@@ -30,7 +30,13 @@ from cloudinit.settings import (PER_ALWAYS)
 LOG = logging.getLogger(__name__)
 
 MERGE_HEADER = 'Merge-Type'
-DEF_MERGERS = mergers.default_mergers()
+
+# Due to the way the loading of yaml configuration was done previously,
+# where previously each cloud config part was appended to a larger yaml
+# file and then finally that file was loaded as one big yaml file we need
+# to mimic that behavior by altering the default strategy to be replacing
+# keys of later mergers.
+DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()')
 
 
 class CloudConfigPartHandler(handlers.Handler):
@@ -53,6 +59,8 @@ class CloudConfigPartHandler(handlers.Handler):
         if self.file_names:
             file_lines.append("# from %s files" % (len(self.file_names)))
             for fn in self.file_names:
+                if not fn:
+                    fn = '?'
                 file_lines.append("# %s" % (fn))
             file_lines.append("")
         if self.cloud_buf is not None:
@@ -111,7 +119,10 @@ class CloudConfigPartHandler(handlers.Handler):
             return
         try:
             self._merge_part(payload, headers)
-            self.file_names.append(filename)
+            # Ensure filename is ok to store
+            for i in ("\n", "\r", "\t"):
+                filename = filename.replace(i, " ")
+            self.file_names.append(filename.strip())
         except:
             util.logexc(LOG, "Failed at merging in cloud config part from %s",
                         filename)
diff --git a/tests/data/merge_sources/expected11.yaml b/tests/data/merge_sources/expected11.yaml
new file mode 100644
index 00000000..c0530dc3
--- /dev/null
+++ b/tests/data/merge_sources/expected11.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+a: 22
+b: 4
+c: 3
diff --git a/tests/data/merge_sources/expected12.yaml b/tests/data/merge_sources/expected12.yaml
new file mode 100644
index 00000000..0421d2c8
--- /dev/null
+++ b/tests/data/merge_sources/expected12.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+a:
+  e:
+    y: 2
diff --git a/tests/data/merge_sources/expected2.yaml b/tests/data/merge_sources/expected2.yaml
index f5312eb1..6eccc2cf 100644
--- a/tests/data/merge_sources/expected2.yaml
+++ b/tests/data/merge_sources/expected2.yaml
@@ -1,3 +1,3 @@
-Blah: 1
+Blah: 3
 Blah2: 2
-Blah3: 3
+Blah3: [1]
diff --git a/tests/data/merge_sources/source11-1.yaml b/tests/data/merge_sources/source11-1.yaml
new file mode 100644
index 00000000..ee29d681
--- /dev/null
+++ b/tests/data/merge_sources/source11-1.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+a: 1
+b: 2
+c: 3
diff --git a/tests/data/merge_sources/source11-2.yaml b/tests/data/merge_sources/source11-2.yaml
new file mode 100644
index 00000000..a9914c34
--- /dev/null
+++ b/tests/data/merge_sources/source11-2.yaml
@@ -0,0 +1,3 @@
+#cloud-config
+
+b: 4
diff --git a/tests/data/merge_sources/source11-3.yaml b/tests/data/merge_sources/source11-3.yaml
new file mode 100644
index 00000000..8f2b8944
--- /dev/null
+++ b/tests/data/merge_sources/source11-3.yaml
@@ -0,0 +1,3 @@
+#cloud-config
+
+a: 22
diff --git a/tests/data/merge_sources/source12-1.yaml b/tests/data/merge_sources/source12-1.yaml
new file mode 100644
index 00000000..09e7c899
--- /dev/null
+++ b/tests/data/merge_sources/source12-1.yaml
@@ -0,0 +1,8 @@
+#cloud-config
+
+a:
+  c: 1
+  d: 2
+  e:
+    z: a
+    y: b
diff --git a/tests/data/merge_sources/source12-2.yaml b/tests/data/merge_sources/source12-2.yaml
new file mode 100644
index 00000000..0421d2c8
--- /dev/null
+++ b/tests/data/merge_sources/source12-2.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+a:
+  e:
+    y: 2
diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index dddf8c6c..ba1c67d7 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -167,6 +167,21 @@ class TestSimpleRun(helpers.ResourceUsingTestCase):
         d = util.mergemanydict([a, b])
         self.assertEquals(c, d)
 
+    def test_compat_merges_dict(self):
+        a = {
+            'Blah': 1,
+            'Blah2': 2,
+            'Blah3': 3,
+        }
+        b = {
+            'Blah': 1,
+            'Blah2': 2,
+            'Blah3': [1],
+        }
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)
+
     def test_compat_merges_list(self):
         a = {'b': [1, 2, 3]}
         b = {'b': [4, 5]}
diff --git a/tests/unittests/test_userdata.py b/tests/unittests/test_userdata.py
index 5fb9acd9..0ebb0484 100644
--- a/tests/unittests/test_userdata.py
+++ b/tests/unittests/test_userdata.py
@@ -108,7 +108,7 @@ p: 1
         contents = util.load_yaml(contents)
         self.assertEquals(contents['run'], ['b', 'c', 'stuff', 'morestuff'])
         self.assertEquals(contents['a'], 'be')
-        self.assertEquals(contents['e'], 'fg')
+        self.assertEquals(contents['e'], [1, 2, 3])
         self.assertEquals(contents['p'], 1)
 
     def test_unhandled_type_warning(self):
-- 
cgit v1.2.3


From b22a82787378e442cb477d2368ccd7653fd95594 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Thu, 9 May 2013 22:47:14 -0700
Subject: Adjust comment on why we are merging cloud config the way we are.

---
 cloudinit/handlers/cloud_config.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py
index 529109ce..c97ca3e8 100644
--- a/cloudinit/handlers/cloud_config.py
+++ b/cloudinit/handlers/cloud_config.py
@@ -35,7 +35,19 @@ MERGE_HEADER = 'Merge-Type'
 # where previously each cloud config part was appended to a larger yaml
 # file and then finally that file was loaded as one big yaml file we need
 # to mimic that behavior by altering the default strategy to be replacing
-# keys of later mergers.
+# keys of prior merges.
+#
+#
+# For example
+# #file 1
+# a: 3
+# #file 2
+# a: 22
+# #combined file (comments not included)
+# a: 3
+# a: 22
+#
+# This gets loaded into yaml with final result {'a': 22}
 DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()')
 
 
-- 
cgit v1.2.3


From 670b46d151477d32056f3fa4eb968c7960f3b472 Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Thu, 9 May 2013 23:04:03 -0700
Subject: Allow lists to have no_replace option.

---
 cloudinit/mergers/m_list.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py
index 1184ded7..76591bea 100644
--- a/cloudinit/mergers/m_list.py
+++ b/cloudinit/mergers/m_list.py
@@ -17,7 +17,7 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 DEF_MERGE_TYPE = 'replace'
-MERGE_TYPES = ('append', 'prepend', DEF_MERGE_TYPE,)
+MERGE_TYPES = ('append', 'prepend', DEF_MERGE_TYPE, 'no_replace')
 
 def _has_any(what, *keys):
     for k in keys:
@@ -67,14 +67,16 @@ class Merger(object):
             return merged_list
 
         def merge_same_index(old_v, new_v):
+            if self._method == 'no_replace':
+                # Leave it be...
+                return old_v
             if isinstance(new_v, (list, tuple)) and self._recurse_array:
                 return self._merger.merge(old_v, new_v)
             if isinstance(new_v, (str, basestring)) and self._recurse_str:
                 return self._merger.merge(old_v, new_v)
             if isinstance(new_v, (dict)) and self._recurse_dict:
                 return self._merger.merge(old_v, new_v)
-            # Otherwise leave it be...
-            return old_v
+            return new_v
 
         # Ok now we are replacing same indexes
         merged_list.extend(value)
-- 
cgit v1.2.3


From 4d83822985bc45dacd611859d76aa5cc3e35e3bc Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Fri, 10 May 2013 14:15:44 -0700
Subject: fix pep8

---
 tests/unittests/test_merging.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index ba1c67d7..4c28f955 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -42,7 +42,7 @@ def _old_mergemanydict(*args):
 
 def _random_str(rand):
     base = ''
-    for _i in xrange(rand.randint(1, 2**8)):
+    for _i in xrange(rand.randint(1, 2 ** 8)):
         base += rand.choice(string.letters + string.digits)
     return base
 
@@ -82,7 +82,7 @@ def _make_dict(current_depth, max_depth, rand):
             if t in [tuple]:
                 base = tuple(base)
     elif t in [long, int]:
-        base = rand.randint(0, 2**8)
+        base = rand.randint(0, 2 ** 8)
     elif t in [str]:
         base = _random_str(rand)
     return base
-- 
cgit v1.2.3


From 9f866ff5540558bab56f10e38481e4ad2efa079b Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Fri, 10 May 2013 14:25:13 -0700
Subject: fix pylint

---
 tests/unittests/test_merging.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index 4c28f955..486b9158 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -11,7 +11,7 @@ import glob
 import os
 import random
 import re
-import string
+import string  # pylint: disable=W0402
 
 SOURCE_PAT = "source*.*yaml"
 EXPECTED_PAT = "expected%s.yaml"
@@ -167,7 +167,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase):
         d = util.mergemanydict([a, b])
         self.assertEquals(c, d)
 
-    def test_compat_merges_dict(self):
+    def test_compat_merges_dict2(self):
         a = {
             'Blah': 1,
             'Blah2': 2,
-- 
cgit v1.2.3