support different and user-suppliable merging algorithms for cloud-config

This adds a very useful mechanism for merging cloud-config, allowing
the user to append to lists (ie, just add more 'run_cmd') or other
things.

See doc/merging.txt for more information, it is intended to be backwards
compatible by default.

LP: #1023179

5 files changed, 129 insertions, 47 deletions

diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py
index 8d6dcd4d..924463ce 100644
--- a/cloudinit/handlers/__init__.py
+++ b/cloudinit/handlers/__init__.py
@@ -27,6 +27,7 @@ from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, FREQUENCIES)
 
 from cloudinit import importer
 from cloudinit import log as logging
+from cloudinit import type_utils
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -69,7 +70,6 @@ INCLUSION_SRCH = sorted(list(INCLUSION_TYPES_MAP.keys()),
 
 
 class Handler(object):
-
     __metaclass__ = abc.ABCMeta
 
     def __init__(self, frequency, version=2):
@@ -77,53 +77,66 @@ class Handler(object):
         self.frequency = frequency
 
     def __repr__(self):
-        return "%s: [%s]" % (util.obj_name(self), self.list_types())
+        return "%s: [%s]" % (type_utils.obj_name(self), self.list_types())
 
     @abc.abstractmethod
     def list_types(self):
         raise NotImplementedError()
 
-    def handle_part(self, data, ctype, filename, payload, frequency):
-        return self._handle_part(data, ctype, filename, payload, frequency)
-
     @abc.abstractmethod
-    def _handle_part(self, data, ctype, filename, payload, frequency):
+    def handle_part(self, *args, **kwargs):
         raise NotImplementedError()
 
 
-def run_part(mod, data, ctype, filename, payload, frequency):
+def run_part(mod, data, filename, payload, frequency, headers):
     mod_freq = mod.frequency
     if not (mod_freq == PER_ALWAYS or
             (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)):
         return
-    mod_ver = mod.handler_version
     # Sanity checks on version (should be an int convertable)
     try:
+        mod_ver = mod.handler_version
         mod_ver = int(mod_ver)
-    except:
+    except (TypeError, ValueError, AttributeError):
         mod_ver = 1
+    content_type = headers['Content-Type']
     try:
         LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s",
-                  mod, ctype, filename, mod_ver, frequency)
-        if mod_ver >= 2:
+                  mod, content_type, filename, mod_ver, frequency)
+        if mod_ver == 3:
+            # Treat as v. 3 which does get a frequency + headers
+            mod.handle_part(data, content_type, filename,
+                            payload, frequency, headers)
+        elif mod_ver == 2:
             # Treat as v. 2 which does get a frequency
-            mod.handle_part(data, ctype, filename, payload, frequency)
-        else:
+            mod.handle_part(data, content_type, filename,
+                            payload, frequency)
+        elif mod_ver == 1:
             # Treat as v. 1 which gets no frequency
-            mod.handle_part(data, ctype, filename, payload)
+            mod.handle_part(data, content_type, filename, payload)
+        else:
+            raise ValueError("Unknown module version %s" % (mod_ver))
     except:
         util.logexc(LOG, ("Failed calling handler %s (%s, %s, %s)"
                          " with frequency %s"),
-                    mod, ctype, filename,
+                    mod, content_type, filename,
                     mod_ver, frequency)
 
 
 def call_begin(mod, data, frequency):
-    run_part(mod, data, CONTENT_START, None, None, frequency)
+    # Create a fake header set
+    headers = {
+        'Content-Type': CONTENT_START,
+    }
+    run_part(mod, data, None, None, frequency, headers)
 
 
 def call_end(mod, data, frequency):
-    run_part(mod, data, CONTENT_END, None, None, frequency)
+    # Create a fake header set
+    headers = {
+        'Content-Type': CONTENT_END,
+    }
+    run_part(mod, data, None, None, frequency, headers)
 
 
 def walker_handle_handler(pdata, _ctype, _filename, payload):
@@ -173,26 +186,27 @@ def _escape_string(text):
     return text
 
 
-def walker_callback(pdata, ctype, filename, payload):
-    if ctype in PART_CONTENT_TYPES:
-        walker_handle_handler(pdata, ctype, filename, payload)
+def walker_callback(data, filename, payload, headers):
+    content_type = headers['Content-Type']
+    if content_type in PART_CONTENT_TYPES:
+        walker_handle_handler(data, content_type, filename, payload)
         return
-    handlers = pdata['handlers']
-    if ctype in pdata['handlers']:
-        run_part(handlers[ctype], pdata['data'], ctype, filename,
-                 payload, pdata['frequency'])
+    handlers = data['handlers']
+    if content_type in handlers:
+        run_part(handlers[content_type], data['data'], filename,
+                 payload, data['frequency'], headers)
     elif payload:
         # Extract the first line or 24 bytes for displaying in the log
         start = _extract_first_or_bytes(payload, 24)
         details = "'%s...'" % (_escape_string(start))
-        if ctype == NOT_MULTIPART_TYPE:
+        if content_type == NOT_MULTIPART_TYPE:
             LOG.warning("Unhandled non-multipart (%s) userdata: %s",
-                        ctype, details)
+                        content_type, details)
         else:
             LOG.warning("Unhandled unknown content-type (%s) userdata: %s",
-                        ctype, details)
+                        content_type, details)
     else:
-        LOG.debug("empty payload of type %s" % ctype)
+        LOG.debug("Empty payload of type %s", content_type)
 
 
 # Callback is a function that will be called with
@@ -212,7 +226,10 @@ def walk(msg, callback, data):
         if not filename:
             filename = PART_FN_TPL % (partnum)
 
-        callback(data, ctype, filename, part.get_payload(decode=True))
+        headers = dict(part)
+        LOG.debug(headers)
+        headers['Content-Type'] = ctype
+        callback(data, filename, part.get_payload(decode=True), headers)
         partnum = partnum + 1
 
 
diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py
index 456b8020..bf2899ab 100644
--- a/cloudinit/handlers/boot_hook.py
+++ b/cloudinit/handlers/boot_hook.py
@@ -56,7 +56,8 @@ class BootHookPartHandler(handlers.Handler):
         util.write_file(filepath, contents, 0700)
         return filepath
 
-    def _handle_part(self, _data, ctype, filename, payload, _frequency):
+    def handle_part(self, _data, ctype, filename,  # pylint: disable=W0221
+                    payload, frequency):  # pylint: disable=W0613
         if ctype in handlers.CONTENT_SIGNALS:
             return
 
diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py
index f6d95244..d30d6338 100644
--- a/cloudinit/handlers/cloud_config.py
+++ b/cloudinit/handlers/cloud_config.py
@@ -22,41 +22,103 @@
 
 from cloudinit import handlers
 from cloudinit import log as logging
+from cloudinit import mergers
 from cloudinit import util
 
 from cloudinit.settings import (PER_ALWAYS)
 
 LOG = logging.getLogger(__name__)
 
+MERGE_HEADER = 'Merge-Type'
+DEF_MERGERS = mergers.default_mergers()
+
 
 class CloudConfigPartHandler(handlers.Handler):
     def __init__(self, paths, **_kwargs):
-        handlers.Handler.__init__(self, PER_ALWAYS)
-        self.cloud_buf = []
+        handlers.Handler.__init__(self, PER_ALWAYS, version=3)
+        self.cloud_buf = None
         self.cloud_fn = paths.get_ipath("cloud_config")
+        self.file_names = []
+        self.mergers = [DEF_MERGERS]
 
     def list_types(self):
         return [
             handlers.type_from_starts_with("#cloud-config"),
         ]
 
-    def _write_cloud_config(self, buf):
+    def _write_cloud_config(self):
         if not self.cloud_fn:
             return
-        lines = [str(b) for b in buf]
-        payload = "\n".join(lines)
-        util.write_file(self.cloud_fn, payload, 0600)
+        # Capture which files we merged from...
+        file_lines = []
+        if self.file_names:
+            file_lines.append("# from %s files" % (len(self.file_names)))
+            for fn in self.file_names:
+                file_lines.append("# %s" % (fn))
+            file_lines.append("")
+        if self.cloud_buf is not None:
+            # Something was actually gathered....
+            lines = [
+                "#cloud-config",
+                '',
+            ]
+            lines.extend(file_lines)
+            lines.append(util.yaml_dumps(self.cloud_buf))
+        else:
+            lines = []
+        util.write_file(self.cloud_fn, "\n".join(lines), 0600)
+
+    def _extract_mergers(self, payload, headers):
+        merge_header_headers = ''
+        for h in [MERGE_HEADER, 'X-%s' % (MERGE_HEADER)]:
+            tmp_h = headers.get(h, '')
+            if tmp_h:
+                merge_header_headers = tmp_h
+                break
+        # Select either the merge-type from the content
+        # or the merge type from the headers or default to our own set
+        # if neither exists (or is empty) from the later.
+        payload_yaml = util.load_yaml(payload)
+        mergers_yaml = mergers.dict_extract_mergers(payload_yaml)
+        mergers_header = mergers.string_extract_mergers(merge_header_headers)
+        all_mergers = []
+        all_mergers.extend(mergers_yaml)
+        all_mergers.extend(mergers_header)
+        if not all_mergers:
+            all_mergers = DEF_MERGERS
+        return all_mergers
+
+    def _merge_part(self, payload, headers):
+        next_mergers = self._extract_mergers(payload, headers)
+        # Use the merger list from the last call, since it is the one
+        # that will be defining how to merge with the next payload.
+        curr_mergers = list(self.mergers[-1])
+        LOG.debug("Merging by applying %s", curr_mergers)
+        self.mergers.append(next_mergers)
+        merger = mergers.construct(curr_mergers)
+        if self.cloud_buf is None:
+            # First time through, merge with an empty dict...
+            self.cloud_buf = {}
+        self.cloud_buf = merger.merge(self.cloud_buf,
+                                      util.load_yaml(payload))
 
-    def _handle_part(self, _data, ctype, filename, payload, _frequency):
+    def _reset(self):
+        self.file_names = []
+        self.cloud_buf = None
+        self.mergers = [DEF_MERGERS]
+
+    def handle_part(self, _data, ctype, filename,  # pylint: disable=W0221
+                    payload, _frequency, headers):  # pylint: disable=W0613
         if ctype == handlers.CONTENT_START:
-            self.cloud_buf = []
+            self._reset()
             return
         if ctype == handlers.CONTENT_END:
-            self._write_cloud_config(self.cloud_buf)
-            self.cloud_buf = []
+            self._write_cloud_config()
+            self._reset()
             return
-
-        filename = util.clean_filename(filename)
-        if not filename:
-            filename = '??'
-        self.cloud_buf.extend(["#%s" % (filename), str(payload)])
+        try:
+            self._merge_part(payload, headers)
+            self.file_names.append(filename)
+        except:
+            util.logexc(LOG, "Failed at merging in cloud config part from %s",
+                        filename)
diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py
index 6c5c11ca..b185c374 100644
--- a/cloudinit/handlers/shell_script.py
+++ b/cloudinit/handlers/shell_script.py
@@ -41,7 +41,8 @@ class ShellScriptPartHandler(handlers.Handler):
             handlers.type_from_starts_with("#!"),
         ]
 
-    def _handle_part(self, _data, ctype, filename, payload, _frequency):
+    def handle_part(self, _data, ctype, filename,  # pylint: disable=W0221
+                    payload, frequency):  # pylint: disable=W0613
         if ctype in handlers.CONTENT_SIGNALS:
             # TODO(harlowja): maybe delete existing things here
             return
diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py
index 0aa7446e..edd56527 100644
--- a/cloudinit/handlers/upstart_job.py
+++ b/cloudinit/handlers/upstart_job.py
@@ -42,7 +42,8 @@ class UpstartJobPartHandler(handlers.Handler):
             handlers.type_from_starts_with("#upstart-job"),
         ]
 
-    def _handle_part(self, _data, ctype, filename, payload, frequency):
+    def handle_part(self, _data, ctype, filename,  # pylint: disable=W0221
+                    payload, frequency):
         if ctype in handlers.CONTENT_SIGNALS:
             return