Add in a bunch of changes and tests.

2 files changed, 62 insertions, 42 deletions

diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py
index 566b61a7..63fdb948 100644
--- a/cloudinit/handlers/__init__.py
+++ b/cloudinit/handlers/__init__.py
@@ -87,7 +87,7 @@ class Handler(object):
         raise NotImplementedError()
 
 
-def run_part(mod, data, filename, payload, headers, frequency):
+def run_part(mod, data, filename, payload, frequency, headers):
     mod_freq = mod.frequency
     if not (mod_freq == PER_ALWAYS or
             (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)):
@@ -98,8 +98,8 @@ def run_part(mod, data, filename, payload, headers, frequency):
         mod_ver = int(mod_ver)
     except (TypeError, ValueError, AttributeError):
         mod_ver = 1
+    content_type = headers['Content-Type']
     try:
-        content_type = headers['Content-Type']
         LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s",
                   mod, content_type, filename, mod_ver, frequency)
         if mod_ver == 3:
@@ -123,17 +123,19 @@ def run_part(mod, data, filename, payload, headers, frequency):
 
 
 def call_begin(mod, data, frequency):
+    # Create a fake header set
     headers = {
         'Content-Type': CONTENT_START,
     }
-    run_part(mod, data, None, None, headers, frequency)
+    run_part(mod, data, None, None, frequency, headers)
 
 
 def call_end(mod, data, frequency):
+    # Create a fake header set
     headers = {
         'Content-Type': CONTENT_END,
     }
-    run_part(mod, data, None, None, headers, frequency)
+    run_part(mod, data, None, None, frequency, headers)
 
 
 def walker_handle_handler(pdata, _ctype, _filename, payload):
@@ -191,12 +193,12 @@ def walker_callback(data, filename, payload, headers):
     handlers = data['handlers']
     if content_type in handlers:
         run_part(handlers[content_type], data['data'], filename,
-                 payload, headers, data['frequency'])
+                 payload, data['frequency'], headers)
     elif payload:
         # Extract the first line or 24 bytes for displaying in the log
         start = _extract_first_or_bytes(payload, 24)
         details = "'%s...'" % (_escape_string(start))
-        if ctype == NOT_MULTIPART_TYPE:
+        if content_type == NOT_MULTIPART_TYPE:
             LOG.warning("Unhandled non-multipart (%s) userdata: %s",
                         content_type, details)
         else:
@@ -224,6 +226,7 @@ def walk(msg, callback, data):
             filename = PART_FN_TPL % (partnum)
 
         headers = dict(part)
+        LOG.debug(headers)
         headers['Content-Type'] = ctype
         callback(data, filename, part.get_payload(decode=True), headers)
         partnum = partnum + 1
diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py
index 02a7ad9d..d458dee2 100644
--- a/cloudinit/handlers/cloud_config.py
+++ b/cloudinit/handlers/cloud_config.py
@@ -29,16 +29,19 @@ from cloudinit.settings import (PER_ALWAYS)
 
 LOG = logging.getLogger(__name__)
 
-DEF_MERGE_TYPE = "list()+dict()+str()"
+DEF_MERGE_TYPE = "list(extend)+dict()+str(append)"
 MERGE_HEADER = 'Merge-Type'
 
 
 class CloudConfigPartHandler(handlers.Handler):
     def __init__(self, paths, **_kwargs):
         handlers.Handler.__init__(self, PER_ALWAYS, version=3)
-        self.cloud_buf = {}
+        self.cloud_buf = None
         self.cloud_fn = paths.get_ipath("cloud_config")
         self.file_names = []
+        self.mergers = [
+            mergers.string_extract_mergers(DEF_MERGE_TYPE),
+        ]
 
     def list_types(self):
         return [
@@ -48,50 +51,64 @@ class CloudConfigPartHandler(handlers.Handler):
     def _write_cloud_config(self, buf):
         if not self.cloud_fn:
             return
-        # Write the combined & merged dictionary/yaml out
-        lines = [
-            "#cloud-config",
-            '',
-        ]
-        # Write which files we merged from
+        # Capture which files we merged from...
+        file_lines = []
         if self.file_names:
-            lines.append("# from %s files" % (len(self.file_names)))
+            file_lines.append("# from %s files" % (len(self.file_names)))
             for fn in self.file_names:
-                lines.append("# %s" % (fn))
-            lines.append("")
-        lines.append(util.yaml_dumps(self.cloud_buf))
+                file_lines.append("# %s" % (fn))
+            file_lines.append("")
+        if self.cloud_buf is not None:
+            lines = [
+                "#cloud-config",
+                '',
+            ]
+            lines.extend(file_lines)
+            lines.append(util.yaml_dumps(self.cloud_buf))
+        else:
+            lines = []
         util.write_file(self.cloud_fn, "\n".join(lines), 0600)
 
-    def _merge_header_extract(self, payload_yaml):
-        merge_header_yaml = ''
-        for k in [MERGE_HEADER, MERGE_HEADER.lower(),
-                  MERGE_HEADER.lower().replace("-", "_")]:
-            if k in payload_yaml:
-                merge_header_yaml = str(payload_yaml[k])
+    def _extract_mergers(self, payload, headers):
+        merge_header_headers = ''
+        for h in [MERGE_HEADER, 'X-%s' % (MERGE_HEADER)]:
+            tmp_h = headers.get(h, '')
+            if tmp_h:
+                merge_header_headers = tmp_h
                 break
-        return merge_header_yaml
-
-    def _merge_part(self, payload, headers):
-        merge_header_headers = headers.get(MERGE_HEADER, '')
-        payload_yaml = util.load_yaml(payload)
-        merge_how = ''
         # Select either the merge-type from the content
         # or the merge type from the headers or default to our own set
-        # if neither exists (or is empty) from the later
-        merge_header_yaml = self._merge_header_extract(payload_yaml)
-        for merge_i in [merge_header_yaml, merge_header_headers]:
-            merge_i = merge_i.strip().lower()
-            if merge_i:
-                merge_how = merge_i
-                break
-        if not merge_how:
-            merge_how = DEF_MERGE_TYPE
-        merger = mergers.construct(merge_how)
-        self.cloud_buf = merger.merge(self.cloud_buf, payload_yaml)
+        # if neither exists (or is empty) from the later.
+        payload_yaml = util.load_yaml(payload)
+        mergers_yaml = mergers.dict_extract_mergers(payload_yaml)
+        mergers_header = mergers.string_extract_mergers(merge_header_headers)
+        all_mergers = []
+        all_mergers.extend(mergers_yaml)
+        all_mergers.extend(mergers_header)
+        if not all_mergers:
+            all_mergers = mergers.string_extract_mergers(DEF_MERGE_TYPE)
+        return all_mergers
+
+    def _merge_part(self, payload, headers):
+        next_mergers = self._extract_mergers(payload, headers)
+        # Use the merger list from the last call, since it is the one
+        # that will be defining how to merge with the next payload.
+        curr_mergers = list(self.mergers[-1])
+        LOG.debug("Merging with %s", curr_mergers)
+        self.mergers.append(next_mergers)
+        merger = mergers.construct(curr_mergers)
+        if self.cloud_buf is None:
+            # First time through, merge with an empty dict...
+            self.cloud_buf = {}
+        self.cloud_buf = merger.merge(self.cloud_buf,
+                                      util.load_yaml(payload))
 
     def _reset(self):
         self.file_names = []
-        self.cloud_buf = {}
+        self.cloud_buf = None
+        self.mergers = [
+            mergers.string_extract_mergers(DEF_MERGE_TYPE),
+        ]
 
     def handle_part(self, _data, ctype, filename, payload, _freq, headers):
         if ctype == handlers.CONTENT_START: