Fixed merging capabilities.

Instead of previously having merging which was not backwards compatible
with the 0.7.1 and prior methods this patch works to ensure said backwards
compatible while at the same time making the new merging functionality
work in a more customizable manner.

39 files changed, 499 insertions, 130 deletions

diff --git a/tests/data/merge_sources/expected1.yaml b/tests/data/merge_sources/expected1.yaml
new file mode 100644
index 00000000..640d282b
--- /dev/null
+++ b/tests/data/merge_sources/expected1.yaml
@@ -0,0 +1 @@
+Blah: ['blah2', 'b']
diff --git a/tests/data/merge_sources/expected10.yaml b/tests/data/merge_sources/expected10.yaml
new file mode 100644
index 00000000..b865db16
--- /dev/null
+++ b/tests/data/merge_sources/expected10.yaml
@@ -0,0 +1,7 @@
+#cloud-config
+
+power_state:
+  delay: 30
+  mode: poweroff
+  message: [Bye, Bye, Pew, Pew]
+  
diff --git a/tests/data/merge_sources/expected11.yaml b/tests/data/merge_sources/expected11.yaml
new file mode 100644
index 00000000..c0530dc3
--- /dev/null
+++ b/tests/data/merge_sources/expected11.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+a: 22
+b: 4
+c: 3
diff --git a/tests/data/merge_sources/expected12.yaml b/tests/data/merge_sources/expected12.yaml
new file mode 100644
index 00000000..0421d2c8
--- /dev/null
+++ b/tests/data/merge_sources/expected12.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+a:
+  e:
+    y: 2
diff --git a/tests/data/merge_sources/expected2.yaml b/tests/data/merge_sources/expected2.yaml
new file mode 100644
index 00000000..6eccc2cf
--- /dev/null
+++ b/tests/data/merge_sources/expected2.yaml
@@ -0,0 +1,3 @@
+Blah: 3
+Blah2: 2
+Blah3: [1]
diff --git a/tests/data/merge_sources/expected3.yaml b/tests/data/merge_sources/expected3.yaml
new file mode 100644
index 00000000..32d9ad48
--- /dev/null
+++ b/tests/data/merge_sources/expected3.yaml
@@ -0,0 +1 @@
+Blah: [blah2, 'blah1']
diff --git a/tests/data/merge_sources/expected4.yaml b/tests/data/merge_sources/expected4.yaml
new file mode 100644
index 00000000..d88d8f73
--- /dev/null
+++ b/tests/data/merge_sources/expected4.yaml
@@ -0,0 +1,2 @@
+#cloud-config
+Blah: {}
diff --git a/tests/data/merge_sources/expected5.yaml b/tests/data/merge_sources/expected5.yaml
new file mode 100644
index 00000000..628f5878
--- /dev/null
+++ b/tests/data/merge_sources/expected5.yaml
@@ -0,0 +1,7 @@
+#cloud-config
+
+Blah: 3
+Blah2: 2
+Blah3: [1]
+
+
diff --git a/tests/data/merge_sources/expected6.yaml b/tests/data/merge_sources/expected6.yaml
new file mode 100644
index 00000000..7afe1d7c
--- /dev/null
+++ b/tests/data/merge_sources/expected6.yaml
@@ -0,0 +1,9 @@
+#cloud-config
+
+run_cmds:
+   - bash
+   - top
+   - ps
+   - vi
+   - emacs
+
diff --git a/tests/data/merge_sources/expected7.yaml b/tests/data/merge_sources/expected7.yaml
new file mode 100644
index 00000000..25284f04
--- /dev/null
+++ b/tests/data/merge_sources/expected7.yaml
@@ -0,0 +1,38 @@
+#cloud-config
+
+users:
+  - default
+  - name: foobar
+    gecos: Foo B. Bar
+    primary-group: foobar
+    groups: users
+    selinux-user: staff_u
+    expiredate: 2012-09-01
+    ssh-import-id: foobar
+    lock-passwd: false
+    passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
+  - name: barfoo
+    gecos: Bar B. Foo
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    groups: users, admin
+    ssh-import-id: None
+    lock-passwd: true
+    ssh-authorized-keys:
+      - <ssh pub key 1>
+      - <ssh pub key 2>
+  - name: cloudy
+    gecos: Magic Cloud App Daemon User
+    inactive: true
+    system: true
+  - bob
+  - joe
+  - sue
+  - name: foobar_jr
+    gecos: Foo B. Bar Jr
+    primary-group: foobar
+    groups: users
+    selinux-user: staff_u
+    expiredate: 2012-09-01
+    ssh-import-id: foobar
+    lock-passwd: false
+    passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
diff --git a/tests/data/merge_sources/expected8.yaml b/tests/data/merge_sources/expected8.yaml
new file mode 100644
index 00000000..69ca562d
--- /dev/null
+++ b/tests/data/merge_sources/expected8.yaml
@@ -0,0 +1,7 @@
+#cloud-config
+
+mounts:
+ - [ ephemeral22, /mnt, auto, "defaults,noexec" ]
+ - [ sdc, /opt/data ]
+ - [ xvdh, /opt/data, "auto", "defaults,nobootwait", "0", "0" ]
+ - [ dd, /dev/zero ]
diff --git a/tests/data/merge_sources/expected9.yaml b/tests/data/merge_sources/expected9.yaml
new file mode 100644
index 00000000..00f91ca0
--- /dev/null
+++ b/tests/data/merge_sources/expected9.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+phone_home:
+ url: http://my.example.com/$INSTANCE_ID/$BLAH_BLAH
+ post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id ]
diff --git a/tests/data/merge_sources/source1-1.yaml b/tests/data/merge_sources/source1-1.yaml
new file mode 100644
index 00000000..38e4e5e0
--- /dev/null
+++ b/tests/data/merge_sources/source1-1.yaml
@@ -0,0 +1,3 @@
+#cloud-config
+Blah: ['blah2']
+
diff --git a/tests/data/merge_sources/source1-2.yaml b/tests/data/merge_sources/source1-2.yaml
new file mode 100644
index 00000000..2cd0e0e5
--- /dev/null
+++ b/tests/data/merge_sources/source1-2.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+Blah: ['b']
+
+merge_how: 'dict(recurse_array,no_replace)+list(append)'
diff --git a/tests/data/merge_sources/source10-1.yaml b/tests/data/merge_sources/source10-1.yaml
new file mode 100644
index 00000000..6ae72a13
--- /dev/null
+++ b/tests/data/merge_sources/source10-1.yaml
@@ -0,0 +1,6 @@
+#cloud-config
+
+power_state:
+ delay: 30
+ mode: poweroff
+ message: [Bye, Bye]
diff --git a/tests/data/merge_sources/source10-2.yaml b/tests/data/merge_sources/source10-2.yaml
new file mode 100644
index 00000000..a38cf1c5
--- /dev/null
+++ b/tests/data/merge_sources/source10-2.yaml
@@ -0,0 +1,6 @@
+#cloud-config
+
+power_state:
+  message: [Pew, Pew]
+  
+merge_how: 'dict(recurse_list)+list(append)'
diff --git a/tests/data/merge_sources/source11-1.yaml b/tests/data/merge_sources/source11-1.yaml
new file mode 100644
index 00000000..ee29d681
--- /dev/null
+++ b/tests/data/merge_sources/source11-1.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+a: 1
+b: 2
+c: 3
diff --git a/tests/data/merge_sources/source11-2.yaml b/tests/data/merge_sources/source11-2.yaml
new file mode 100644
index 00000000..a9914c34
--- /dev/null
+++ b/tests/data/merge_sources/source11-2.yaml
@@ -0,0 +1,3 @@
+#cloud-config
+
+b: 4
diff --git a/tests/data/merge_sources/source11-3.yaml b/tests/data/merge_sources/source11-3.yaml
new file mode 100644
index 00000000..8f2b8944
--- /dev/null
+++ b/tests/data/merge_sources/source11-3.yaml
@@ -0,0 +1,3 @@
+#cloud-config
+
+a: 22
diff --git a/tests/data/merge_sources/source12-1.yaml b/tests/data/merge_sources/source12-1.yaml
new file mode 100644
index 00000000..09e7c899
--- /dev/null
+++ b/tests/data/merge_sources/source12-1.yaml
@@ -0,0 +1,8 @@
+#cloud-config
+
+a:
+  c: 1
+  d: 2
+  e:
+    z: a
+    y: b
diff --git a/tests/data/merge_sources/source12-2.yaml b/tests/data/merge_sources/source12-2.yaml
new file mode 100644
index 00000000..0421d2c8
--- /dev/null
+++ b/tests/data/merge_sources/source12-2.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+a:
+  e:
+    y: 2
diff --git a/tests/data/merge_sources/source2-1.yaml b/tests/data/merge_sources/source2-1.yaml
new file mode 100644
index 00000000..c7a33aaa
--- /dev/null
+++ b/tests/data/merge_sources/source2-1.yaml
@@ -0,0 +1,6 @@
+#cloud-config
+
+
+Blah: 1
+Blah2: 2
+Blah3: 3
diff --git a/tests/data/merge_sources/source2-2.yaml b/tests/data/merge_sources/source2-2.yaml
new file mode 100644
index 00000000..8f2fdc1a
--- /dev/null
+++ b/tests/data/merge_sources/source2-2.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+Blah: 3
+Blah2: 2
+Blah3: [1]
diff --git a/tests/data/merge_sources/source3-1.yaml b/tests/data/merge_sources/source3-1.yaml
new file mode 100644
index 00000000..2303e906
--- /dev/null
+++ b/tests/data/merge_sources/source3-1.yaml
@@ -0,0 +1,4 @@
+#cloud-config
+Blah: ['blah1']
+
+
diff --git a/tests/data/merge_sources/source3-2.yaml b/tests/data/merge_sources/source3-2.yaml
new file mode 100644
index 00000000..dca2ad10
--- /dev/null
+++ b/tests/data/merge_sources/source3-2.yaml
@@ -0,0 +1,4 @@
+#cloud-config
+Blah: ['blah2']
+
+merge_how: 'dict(recurse_array,no_replace)+list(prepend)'
diff --git a/tests/data/merge_sources/source4-1.yaml b/tests/data/merge_sources/source4-1.yaml
new file mode 100644
index 00000000..e5b16872
--- /dev/null
+++ b/tests/data/merge_sources/source4-1.yaml
@@ -0,0 +1,3 @@
+#cloud-config
+Blah:
+  b: 1
diff --git a/tests/data/merge_sources/source4-2.yaml b/tests/data/merge_sources/source4-2.yaml
new file mode 100644
index 00000000..1844e0f8
--- /dev/null
+++ b/tests/data/merge_sources/source4-2.yaml
@@ -0,0 +1,6 @@
+#cloud-config
+Blah:
+  b: null
+
+
+merge_how: 'dict(allow_delete,no_replace)+list()'
diff --git a/tests/data/merge_sources/source5-1.yaml b/tests/data/merge_sources/source5-1.yaml
new file mode 100644
index 00000000..c7a33aaa
--- /dev/null
+++ b/tests/data/merge_sources/source5-1.yaml
@@ -0,0 +1,6 @@
+#cloud-config
+
+
+Blah: 1
+Blah2: 2
+Blah3: 3
diff --git a/tests/data/merge_sources/source5-2.yaml b/tests/data/merge_sources/source5-2.yaml
new file mode 100644
index 00000000..f61c96a2
--- /dev/null
+++ b/tests/data/merge_sources/source5-2.yaml
@@ -0,0 +1,8 @@
+#cloud-config
+
+Blah: 3
+Blah2: 2
+Blah3: [1]
+
+
+merge_how: 'dict(replace)+list(append)'
diff --git a/tests/data/merge_sources/source6-1.yaml b/tests/data/merge_sources/source6-1.yaml
new file mode 100644
index 00000000..519f7309
--- /dev/null
+++ b/tests/data/merge_sources/source6-1.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+run_cmds:
+  - bash
+  - top
diff --git a/tests/data/merge_sources/source6-2.yaml b/tests/data/merge_sources/source6-2.yaml
new file mode 100644
index 00000000..d8fac446
--- /dev/null
+++ b/tests/data/merge_sources/source6-2.yaml
@@ -0,0 +1,8 @@
+#cloud-config
+
+run_cmds:
+  - ps
+  - vi
+  - emacs
+
+merge_type: 'list(append)+dict(recurse_array)+str()'
diff --git a/tests/data/merge_sources/source7-1.yaml b/tests/data/merge_sources/source7-1.yaml
new file mode 100644
index 00000000..8fb9b32a
--- /dev/null
+++ b/tests/data/merge_sources/source7-1.yaml
@@ -0,0 +1,27 @@
+#cloud-config
+
+users:
+  - default
+  - name: foobar
+    gecos: Foo B. Bar
+    primary-group: foobar
+    groups: users
+    selinux-user: staff_u
+    expiredate: 2012-09-01
+    ssh-import-id: foobar
+    lock-passwd: false
+    passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
+  - name: barfoo
+    gecos: Bar B. Foo
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    groups: users, admin
+    ssh-import-id: None
+    lock-passwd: true
+    ssh-authorized-keys:
+      - <ssh pub key 1>
+      - <ssh pub key 2>
+  - name: cloudy
+    gecos: Magic Cloud App Daemon User
+    inactive: true
+    system: true
+
diff --git a/tests/data/merge_sources/source7-2.yaml b/tests/data/merge_sources/source7-2.yaml
new file mode 100644
index 00000000..1e26201b
--- /dev/null
+++ b/tests/data/merge_sources/source7-2.yaml
@@ -0,0 +1,17 @@
+#cloud-config
+
+users:
+  - bob
+  - joe
+  - sue
+  - name: foobar_jr
+    gecos: Foo B. Bar Jr
+    primary-group: foobar
+    groups: users
+    selinux-user: staff_u
+    expiredate: 2012-09-01
+    ssh-import-id: foobar
+    lock-passwd: false
+    passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
+
+merge_how: "dict(recurse_array)+list(append)"
diff --git a/tests/data/merge_sources/source8-1.yaml b/tests/data/merge_sources/source8-1.yaml
new file mode 100644
index 00000000..5ea51c2c
--- /dev/null
+++ b/tests/data/merge_sources/source8-1.yaml
@@ -0,0 +1,7 @@
+#cloud-config
+
+mounts:
+ - [ ephemeral0, /mnt, auto, "defaults,noexec" ]
+ - [ sdc, /opt/data ]
+ - [ xvdh, /opt/data, "auto", "defaults,nobootwait", "0", "0" ]
+ - [ dd, /dev/zero ]
diff --git a/tests/data/merge_sources/source8-2.yaml b/tests/data/merge_sources/source8-2.yaml
new file mode 100644
index 00000000..7fa3262b
--- /dev/null
+++ b/tests/data/merge_sources/source8-2.yaml
@@ -0,0 +1,6 @@
+#cloud-config
+
+mounts:
+ - [ ephemeral22, /mnt, auto, "defaults,noexec" ]
+
+merge_how: 'dict(recurse_array)+list(recurse_list,recurse_str)+str()'
diff --git a/tests/data/merge_sources/source9-1.yaml b/tests/data/merge_sources/source9-1.yaml
new file mode 100644
index 00000000..0b102ba6
--- /dev/null
+++ b/tests/data/merge_sources/source9-1.yaml
@@ -0,0 +1,5 @@
+#cloud-config
+
+phone_home:
+ url: http://my.example.com/$INSTANCE_ID/
+ post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id ]
diff --git a/tests/data/merge_sources/source9-2.yaml b/tests/data/merge_sources/source9-2.yaml
new file mode 100644
index 00000000..ac85afc6
--- /dev/null
+++ b/tests/data/merge_sources/source9-2.yaml
@@ -0,0 +1,6 @@
+#cloud-config
+
+phone_home:
+ url: $BLAH_BLAH
+
+merge_how: 'dict(recurse_str)+str(append)'
diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index ad137e85..486b9158 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -1,142 +1,255 @@
 from tests.unittests import helpers
 
-from cloudinit import mergers
+from cloudinit.handlers import cloud_config
+from cloudinit.handlers import (CONTENT_START, CONTENT_END)
 
+from cloudinit import helpers as c_helpers
+from cloudinit import util
 
-class TestSimpleRun(helpers.MockerTestCase):
-    def test_basic_merge(self):
-        source = {
-            'Blah': ['blah2'],
-            'Blah3': 'c',
-        }
-        merge_with = {
-            'Blah2': ['blah3'],
-            'Blah3': 'b',
-            'Blah': ['123'],
+import collections
+import glob
+import os
+import random
+import re
+import string  # pylint: disable=W0402
+
+SOURCE_PAT = "source*.*yaml"
+EXPECTED_PAT = "expected%s.yaml"
+TYPES = [long, int, dict, str, list, tuple, None]
+
+
+def _old_mergedict(src, cand):
+    """
+    Merge values from C{cand} into C{src}.
+    If C{src} has a key C{cand} will not override.
+    Nested dictionaries are merged recursively.
+    """
+    if isinstance(src, dict) and isinstance(cand, dict):
+        for (k, v) in cand.iteritems():
+            if k not in src:
+                src[k] = v
+            else:
+                src[k] = _old_mergedict(src[k], v)
+    return src
+
+
+def _old_mergemanydict(*args):
+    out = {}
+    for a in args:
+        out = _old_mergedict(out, a)
+    return out
+
+
+def _random_str(rand):
+    base = ''
+    for _i in xrange(rand.randint(1, 2 ** 8)):
+        base += rand.choice(string.letters + string.digits)
+    return base
+
+
+class _NoMoreException(Exception):
+    pass
+
+
+def _make_dict(current_depth, max_depth, rand):
+    if current_depth >= max_depth:
+        raise _NoMoreException()
+    if current_depth == 0:
+        t = dict
+    else:
+        t = rand.choice(TYPES)
+    base = None
+    if t in [None]:
+        return base
+    if t in [dict, list, tuple]:
+        if t in [dict]:
+            amount = rand.randint(0, 5)
+            keys = [_random_str(rand) for _i in xrange(0, amount)]
+            base = {}
+            for k in keys:
+                try:
+                    base[k] = _make_dict(current_depth + 1, max_depth, rand)
+                except _NoMoreException:
+                    pass
+        elif t in [list, tuple]:
+            base = []
+            amount = rand.randint(0, 5)
+            for _i in xrange(0, amount):
+                try:
+                    base.append(_make_dict(current_depth + 1, max_depth, rand))
+                except _NoMoreException:
+                    pass
+            if t in [tuple]:
+                base = tuple(base)
+    elif t in [long, int]:
+        base = rand.randint(0, 2 ** 8)
+    elif t in [str]:
+        base = _random_str(rand)
+    return base
+
+
+def make_dict(max_depth, seed=None):
+    max_depth = max(1, max_depth)
+    rand = random.Random(seed)
+    return _make_dict(0, max_depth, rand)
+
+
+class TestSimpleRun(helpers.ResourceUsingTestCase):
+    def _load_merge_files(self):
+        merge_root = self.resourceLocation('merge_sources')
+        tests = []
+        source_ids = collections.defaultdict(list)
+        expected_files = {}
+        for fn in glob.glob(os.path.join(merge_root, SOURCE_PAT)):
+            base_fn = os.path.basename(fn)
+            file_id = re.match(r"source(\d+)\-(\d+)[.]yaml", base_fn)
+            if not file_id:
+                raise IOError("File %s does not have a numeric identifier"
+                              % (fn))
+            file_id = int(file_id.group(1))
+            source_ids[file_id].append(fn)
+            expected_fn = os.path.join(merge_root, EXPECTED_PAT % (file_id))
+            if not os.path.isfile(expected_fn):
+                raise IOError("No expected file found at %s" % (expected_fn))
+            expected_files[file_id] = expected_fn
+        for i in sorted(source_ids.keys()):
+            source_file_contents = []
+            for fn in sorted(source_ids[i]):
+                source_file_contents.append([fn, util.load_file(fn)])
+            expected = util.load_yaml(util.load_file(expected_files[i]))
+            entry = [source_file_contents, [expected, expected_files[i]]]
+            tests.append(entry)
+        return tests
+
+    def test_seed_runs(self):
+        test_dicts = []
+        for i in range(1, 50):
+            base_dicts = []
+            for j in range(1, 50):
+                base_dicts.append(make_dict(5, i * j))
+            test_dicts.append(base_dicts)
+        for test in test_dicts:
+            c = _old_mergemanydict(*test)
+            d = util.mergemanydict(test)
+            self.assertEquals(c, d)
+
+    def test_merge_cc_samples(self):
+        tests = self._load_merge_files()
+        paths = c_helpers.Paths({})
+        cc_handler = cloud_config.CloudConfigPartHandler(paths)
+        cc_handler.cloud_fn = None
+        for (payloads, (expected_merge, expected_fn)) in tests:
+            cc_handler.handle_part(None, CONTENT_START, None,
+                                   None, None, None)
+            merging_fns = []
+            for (fn, contents) in payloads:
+                cc_handler.handle_part(None, None, "%s.yaml" % (fn),
+                                       contents, None, {})
+                merging_fns.append(fn)
+            merged_buf = cc_handler.cloud_buf
+            cc_handler.handle_part(None, CONTENT_END, None,
+                                   None, None, None)
+            fail_msg = "Equality failure on checking %s with %s: %s != %s"
+            fail_msg = fail_msg % (expected_fn,
+                                   ",".join(merging_fns), merged_buf,
+                                   expected_merge)
+            self.assertEquals(expected_merge, merged_buf, msg=fail_msg)
+
+    def test_compat_merges_dict(self):
+        a = {
+            '1': '2',
+            'b': 'c',
         }
-        # Basic merge should not do thing special
-        merge_how = "list()+dict()+str()"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['Blah'], ['blah2'])
-        self.assertEquals(merged['Blah2'], ['blah3'])
-        self.assertEquals(merged['Blah3'], 'c')
-
-    def test_dict_overwrite(self):
-        source = {
-            'Blah': ['blah2'],
+        b = {
+            'b': 'e',
         }
-        merge_with = {
-            'Blah': ['123'],
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)
+
+    def test_compat_merges_dict2(self):
+        a = {
+            'Blah': 1,
+            'Blah2': 2,
+            'Blah3': 3,
         }
-        # Now lets try a dict overwrite
-        merge_how = "list()+dict(overwrite)+str()"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['Blah'], ['123'])
-
-    def test_string_append(self):
-        source = {
-            'Blah': 'blah2',
+        b = {
+            'Blah': 1,
+            'Blah2': 2,
+            'Blah3': [1],
         }
-        merge_with = {
-            'Blah': '345',
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)
+
+    def test_compat_merges_list(self):
+        a = {'b': [1, 2, 3]}
+        b = {'b': [4, 5]}
+        c = {'b': [6, 7]}
+        e = _old_mergemanydict(a, b, c)
+        f = util.mergemanydict([a, b, c])
+        self.assertEquals(e, f)
+
+    def test_compat_merges_str(self):
+        a = {'b': "hi"}
+        b = {'b': "howdy"}
+        c = {'b': "hallo"}
+        e = _old_mergemanydict(a, b, c)
+        f = util.mergemanydict([a, b, c])
+        self.assertEquals(e, f)
+
+    def test_compat_merge_sub_dict(self):
+        a = {
+            '1': '2',
+            'b': {
+                'f': 'g',
+                'e': 'c',
+                'h': 'd',
+                'hh': {
+                    '1': 2,
+                },
+            }
         }
-        merge_how = "list()+dict()+str(append)"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['Blah'], 'blah2345')
-
-    def test_list_extend(self):
-        source = ['abc']
-        merge_with = ['123']
-        merge_how = "list(extend)+dict()+str()"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged, ['abc', '123'])
-
-    def test_deep_merge(self):
-        source = {
-            'a': [1, 'b', 2],
-            'b': 'blahblah',
-            'c': {
-                'e': [1, 2, 3],
-                'f': 'bigblobof',
-                'iamadict': {
-                    'ok': 'ok',
+        b = {
+            'b': {
+                'e': 'c',
+                'hh': {
+                    '3': 4,
                 }
-            },
-            'run': [
-                'runme',
-                'runme2',
-            ],
-            'runmereally': [
-                'e', ['a'], 'd',
-            ],
+            }
         }
-        merge_with = {
-            'a': ['e', 'f', 'g'],
-            'b': 'more',
-            'c': {
-                'a': 'b',
-                'f': 'stuff',
-            },
-            'run': [
-                'morecmd',
-                'moremoremore',
-            ],
-            'runmereally': [
-                'blah', ['b'], 'e',
-            ],
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)
+
+    def test_compat_merge_sub_dict2(self):
+        a = {
+            '1': '2',
+            'b': {
+                'f': 'g',
+            }
         }
-        merge_how = "list(extend)+dict()+str(append)"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['a'], [1, 'b', 2, 'e', 'f', 'g'])
-        self.assertEquals(merged['b'], 'blahblahmore')
-        self.assertEquals(merged['c']['f'], 'bigblobofstuff')
-        self.assertEquals(merged['run'], ['runme', 'runme2', 'morecmd',
-                                          'moremoremore'])
-        self.assertEquals(merged['runmereally'], ['e', ['a'], 'd', 'blah',
-                                                  ['b'], 'e'])
-
-    def test_dict_overwrite_layered(self):
-        source = {
-            'Blah3': {
-                'f': '3',
-                'g': {
-                    'a': 'b',
-                }
+        b = {
+            'b': {
+                'e': 'c',
             }
         }
-        merge_with = {
-            'Blah3': {
-                'e': '2',
-                'g': {
-                    'e': 'f',
-                }
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)
+
+    def test_compat_merge_sub_list(self):
+        a = {
+            '1': '2',
+            'b': {
+                'f': ['1'],
             }
         }
-        merge_how = "list()+dict()+str()"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['Blah3'], {
-                'e': '2',
-                'f': '3',
-                'g': {
-                    'a': 'b',
-                    'e': 'f',
-                }
-        })
+        b = {
+            'b': {
+                'f': [],
+            }
+        }
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)
diff --git a/tests/unittests/test_userdata.py b/tests/unittests/test_userdata.py
index fdfe2542..0ebb0484 100644
--- a/tests/unittests/test_userdata.py
+++ b/tests/unittests/test_userdata.py
@@ -60,7 +60,6 @@ run:
  - c
 '''
         message1 = MIMEBase("text", "cloud-config")
-        message1['Merge-Type'] = 'dict()+list(extend)+str(append)'
         message1.set_payload(blob)
 
         blob2 = '''
@@ -72,7 +71,8 @@ run:
  - morestuff
 '''
         message2 = MIMEBase("text", "cloud-config")
-        message2['X-Merge-Type'] = 'dict()+list(extend)+str()'
+        message2['X-Merge-Type'] = ('dict(recurse_array,'
+                                    'recurse_str)+list(append)+str(append)')
         message2.set_payload(blob2)
 
         blob3 = '''
@@ -84,7 +84,6 @@ e:
 p: 1
 '''
         message3 = MIMEBase("text", "cloud-config")
-        message3['Merge-Type'] = 'dict()+list()+str()'
         message3.set_payload(blob3)
 
         messages = [message1, message2, message3]
@@ -109,7 +108,7 @@ p: 1
         contents = util.load_yaml(contents)
         self.assertEquals(contents['run'], ['b', 'c', 'stuff', 'morestuff'])
         self.assertEquals(contents['a'], 'be')
-        self.assertEquals(contents['e'], 'fg')
+        self.assertEquals(contents['e'], [1, 2, 3])
         self.assertEquals(contents['p'], 1)
 
     def test_unhandled_type_warning(self):