summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJoshua Harlow <harlowja@gmail.com>2012-11-20 20:02:48 -0800
committerJoshua Harlow <harlowja@gmail.com>2012-11-20 20:02:48 -0800
commit8bc85abd97e06d964bbd26208eb732e80eb87c10 (patch)
tree6cbff248cbe85d3ef26da32bc185f34edd8f5c4d
parentd324a2cb0b10a4cd1b1b05dd23d0040ab3e9621c (diff)
downloadvyos-cloud-init-8bc85abd97e06d964bbd26208eb732e80eb87c10.tar.gz
vyos-cloud-init-8bc85abd97e06d964bbd26208eb732e80eb87c10.zip
Start allowing different merging types to be applied
After user data handling splits apart all the different content types into there various mime messages it is nice to be able to have each message specify how it should be merged (mainly for cloud-config or cloud-archive) into the single cloud config that is eventually used. This starts to add a plugable merging framework and the needed components to activate said headers and merging.
-rw-r--r--cloudinit/handlers/__init__.py49
-rw-r--r--cloudinit/handlers/boot_hook.py2
-rw-r--r--cloudinit/handlers/cloud_config.py22
-rw-r--r--cloudinit/handlers/shell_script.py2
-rw-r--r--cloudinit/handlers/upstart_job.py2
-rw-r--r--cloudinit/mergers/__init__.py104
-rw-r--r--cloudinit/mergers/dict.py33
-rw-r--r--cloudinit/mergers/list.py41
-rw-r--r--cloudinit/mergers/str.py28
9 files changed, 246 insertions, 37 deletions
diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py
index 8d6dcd4d..bfccfd89 100644
--- a/cloudinit/handlers/__init__.py
+++ b/cloudinit/handlers/__init__.py
@@ -69,7 +69,6 @@ INCLUSION_SRCH = sorted(list(INCLUSION_TYPES_MAP.keys()),
class Handler(object):
-
__metaclass__ = abc.ABCMeta
def __init__(self, frequency, version=2):
@@ -83,15 +82,12 @@ class Handler(object):
def list_types(self):
raise NotImplementedError()
- def handle_part(self, data, ctype, filename, payload, frequency):
- return self._handle_part(data, ctype, filename, payload, frequency)
-
@abc.abstractmethod
- def _handle_part(self, data, ctype, filename, payload, frequency):
+ def handle_part(self, *args, **kwargs):
raise NotImplementedError()
-def run_part(mod, data, ctype, filename, payload, frequency):
+def run_part(mod, data, filename, payload, headers, frequency):
mod_freq = mod.frequency
if not (mod_freq == PER_ALWAYS or
(frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)):
@@ -102,19 +98,25 @@ def run_part(mod, data, ctype, filename, payload, frequency):
mod_ver = int(mod_ver)
except:
mod_ver = 1
+ content_type = headers['Content-Type']
try:
LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s",
- mod, ctype, filename, mod_ver, frequency)
- if mod_ver >= 2:
+ mod, content_type, filename, mod_ver, frequency)
+ if mod_ver == 3:
+ # Treat as v. 3 which does get a frequency + headers
+ mod.handle_part(data, content_type, filename,
+ payload, frequency, headers)
+ elif mod_ver == 2:
# Treat as v. 2 which does get a frequency
- mod.handle_part(data, ctype, filename, payload, frequency)
+ mod.handle_part(data, content_type, filename,
+ payload, frequency)
else:
# Treat as v. 1 which gets no frequency
- mod.handle_part(data, ctype, filename, payload)
+ mod.handle_part(data, content_type, filename, payload)
except:
util.logexc(LOG, ("Failed calling handler %s (%s, %s, %s)"
" with frequency %s"),
- mod, ctype, filename,
+ mod, content_type, filename,
mod_ver, frequency)
@@ -173,26 +175,27 @@ def _escape_string(text):
return text
-def walker_callback(pdata, ctype, filename, payload):
- if ctype in PART_CONTENT_TYPES:
- walker_handle_handler(pdata, ctype, filename, payload)
+def walker_callback(data, filename, payload, headers):
+ content_type = headers['Content-Type']
+ if content_type in PART_CONTENT_TYPES:
+ walker_handle_handler(data, content_type, filename, payload)
return
- handlers = pdata['handlers']
- if ctype in pdata['handlers']:
- run_part(handlers[ctype], pdata['data'], ctype, filename,
- payload, pdata['frequency'])
+ handlers = data['handlers']
+ if content_type in handlers:
+ run_part(handlers[content_type], data['data'], filename,
+ payload, headers, data['frequency'])
elif payload:
# Extract the first line or 24 bytes for displaying in the log
start = _extract_first_or_bytes(payload, 24)
details = "'%s...'" % (_escape_string(start))
if ctype == NOT_MULTIPART_TYPE:
LOG.warning("Unhandled non-multipart (%s) userdata: %s",
- ctype, details)
+ content_type, details)
else:
LOG.warning("Unhandled unknown content-type (%s) userdata: %s",
- ctype, details)
+ content_type, details)
else:
- LOG.debug("empty payload of type %s" % ctype)
+ LOG.debug("Empty payload of type %s", content_type)
# Callback is a function that will be called with
@@ -212,7 +215,9 @@ def walk(msg, callback, data):
if not filename:
filename = PART_FN_TPL % (partnum)
- callback(data, ctype, filename, part.get_payload(decode=True))
+ callback(data, ctype, filename,
+ part.get_payload(decode=True),
+ dict(part))
partnum = partnum + 1
diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py
index 456b8020..bf313f10 100644
--- a/cloudinit/handlers/boot_hook.py
+++ b/cloudinit/handlers/boot_hook.py
@@ -56,7 +56,7 @@ class BootHookPartHandler(handlers.Handler):
util.write_file(filepath, contents, 0700)
return filepath
- def _handle_part(self, _data, ctype, filename, payload, _frequency):
+ def handle_part(self, _data, ctype, filename, payload, _frequency):
if ctype in handlers.CONTENT_SIGNALS:
return
diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py
index f6d95244..86027187 100644
--- a/cloudinit/handlers/cloud_config.py
+++ b/cloudinit/handlers/cloud_config.py
@@ -22,6 +22,7 @@
from cloudinit import handlers
from cloudinit import log as logging
+from cloudinit import mergers
from cloudinit import util
from cloudinit.settings import (PER_ALWAYS)
@@ -31,8 +32,8 @@ LOG = logging.getLogger(__name__)
class CloudConfigPartHandler(handlers.Handler):
def __init__(self, paths, **_kwargs):
- handlers.Handler.__init__(self, PER_ALWAYS)
- self.cloud_buf = []
+ handlers.Handler.__init__(self, PER_ALWAYS, version=3)
+ self.cloud_buf = {}
self.cloud_fn = paths.get_ipath("cloud_config")
def list_types(self):
@@ -43,20 +44,17 @@ class CloudConfigPartHandler(handlers.Handler):
def _write_cloud_config(self, buf):
if not self.cloud_fn:
return
- lines = [str(b) for b in buf]
- payload = "\n".join(lines)
+ payload = util.yaml_dumps(self.cloud_buf)
util.write_file(self.cloud_fn, payload, 0600)
- def _handle_part(self, _data, ctype, filename, payload, _frequency):
+ def handle_part(self, _data, ctype, filename, payload, _frequency, headers):
if ctype == handlers.CONTENT_START:
- self.cloud_buf = []
+ self.cloud_buf = {}
return
if ctype == handlers.CONTENT_END:
self._write_cloud_config(self.cloud_buf)
- self.cloud_buf = []
+ self.cloud_buf = {}
return
-
- filename = util.clean_filename(filename)
- if not filename:
- filename = '??'
- self.cloud_buf.extend(["#%s" % (filename), str(payload)])
+ merge_how = headers.get("Merge-Type", 'list+dict+str')
+ merger = mergers.construct(merge_how)
+ self.cloud_buf = merger.merge(self.cloud_buf, util.load_yaml(payload))
diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py
index 6c5c11ca..2a87e8dd 100644
--- a/cloudinit/handlers/shell_script.py
+++ b/cloudinit/handlers/shell_script.py
@@ -41,7 +41,7 @@ class ShellScriptPartHandler(handlers.Handler):
handlers.type_from_starts_with("#!"),
]
- def _handle_part(self, _data, ctype, filename, payload, _frequency):
+ def handle_part(self, _data, ctype, filename, payload, _frequency):
if ctype in handlers.CONTENT_SIGNALS:
# TODO(harlowja): maybe delete existing things here
return
diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py
index 99e0afde..a5cb9b0c 100644
--- a/cloudinit/handlers/upstart_job.py
+++ b/cloudinit/handlers/upstart_job.py
@@ -42,7 +42,7 @@ class UpstartJobPartHandler(handlers.Handler):
handlers.type_from_starts_with("#upstart-job"),
]
- def _handle_part(self, _data, ctype, filename, payload, frequency):
+ def handle_part(self, _data, ctype, filename, payload, frequency):
if ctype in handlers.CONTENT_SIGNALS:
return
diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py
new file mode 100644
index 00000000..b3e728b0
--- /dev/null
+++ b/cloudinit/mergers/__init__.py
@@ -0,0 +1,104 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+from cloudinit import importer
+from cloudinit import log as logging
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+class UnknownMerger(object):
+ # Named differently so auto-method finding
+ # doesn't pick this up if there is ever a type
+ # named "unknown"
+ def _handle_unknown(self, meth_wanted, value, merge_with):
+ return value
+
+ def merge(self, source, merge_with):
+ type_name = util.obj_name(source)
+ type_name = type_name.lower()
+ method_name = "_on_%s" % (type_name)
+ meth = None
+ args = [source, merge_with]
+ if hasattr(self, method_name):
+ meth = getattr(self, method_name)
+ if not meth:
+ meth = self._handle_unknown
+ args.insert(0, method_name)
+ return meth(*args)
+
+
+class LookupMerger(UnknownMerger):
+ def __init__(self, lookups=None):
+ UnknownMerger.__init__(self)
+ if lookups is None:
+ self._lookups = []
+ else:
+ self._lookups = lookups
+
+ def _handle_unknown(self, meth_wanted, value, merge_with):
+ meth = None
+ for merger in self._lookups:
+ if hasattr(merger, meth_wanted):
+ # First one that has that method/attr gets to be
+ # the one that will be called
+ meth = getattr(merger, meth_wanted)
+ break
+ if not meth:
+ return UnknownMerger._handle_unknown(self, meth_wanted,
+ value, merge_with)
+ return meth(value, merge_with)
+
+
+def _extract_merger_names(merge_how):
+ names = []
+ for m_name in merge_how.split("+"):
+ # Canonicalize the name (so that it can be found
+ # even when users alter it in various ways...
+ m_name = m_name.lower().strip()
+ m_name = m_name.replace(" ", "_")
+ m_name = m_name.replace("\t", "_")
+ m_name = m_name.replace("-", "_")
+ if not m_name:
+ continue
+ names.append(m_name)
+ return names
+
+
+def construct(merge_how, default_classes=None):
+ mergers = []
+ merger_classes = []
+ root = LookupMerger(mergers)
+ for m_name in _extract_merger_names(merge_how):
+ merger_locs = importer.find_module(m_name,
+ [__name__],
+ ['Merger'])
+ if not merger_locs:
+ msg = "Could not find merger named %s" % (m_name)
+ raise ImportError(msg)
+ else:
+ mod = importer.import_module(merger_locs[0])
+ cls = getattr(mod, 'Merger')
+ merger_classes.append(cls)
+ if not merger_classes and default_classes:
+ merger_classes = default_classes
+ for m_class in merger_classes:
+ mergers.append(m_class(root))
+ return root \ No newline at end of file
diff --git a/cloudinit/mergers/dict.py b/cloudinit/mergers/dict.py
new file mode 100644
index 00000000..a0ffaa33
--- /dev/null
+++ b/cloudinit/mergers/dict.py
@@ -0,0 +1,33 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+class Merger(object):
+ def __init__(self, merger):
+ self._merger = merger
+
+ def _on_dict(self, value, merge_with):
+ if not isinstance(merge_with, (dict)):
+ return value
+ merged = dict(value)
+ for (k, v) in merge_with.items():
+ if k in merged:
+ merged[k] = self._merger.merge(merged[k], v)
+ else:
+ merged[k] = v
+ return merged
diff --git a/cloudinit/mergers/list.py b/cloudinit/mergers/list.py
new file mode 100644
index 00000000..ad1b9793
--- /dev/null
+++ b/cloudinit/mergers/list.py
@@ -0,0 +1,41 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+class Merger(object):
+ def __init__(self, merger):
+ self._merger = merger
+
+ def _on_tuple(self, value, merge_with):
+ return self._on_list(list(value), merge_with)
+
+ def _on_list(self, value, merge_with):
+ if isinstance(merge_with, (tuple, list)):
+ new_value = list(value)
+ for m_v in merge_with:
+ m_am = 0
+ for (i, o_v) in enumerate(new_value):
+ if m_v == o_v:
+ new_value[i] = self._merger.merge(o_v, m_v)
+ m_am += 1
+ if m_am == 0:
+ new_value.append(m_v)
+ else:
+ new_value = list(value)
+ new_value.append(merge_with)
+ return new_value
diff --git a/cloudinit/mergers/str.py b/cloudinit/mergers/str.py
new file mode 100644
index 00000000..7c3fa585
--- /dev/null
+++ b/cloudinit/mergers/str.py
@@ -0,0 +1,28 @@
+# vi: ts=4 expandtab
+#
+# Copyright (C) 2012 Yahoo! Inc.
+#
+# Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+class Merger(object):
+ def __init__(self, merger):
+ pass
+
+ def _on_unicode(self, value, merge_with):
+ return self._on_str(value, merge_with)
+
+ def _on_str(self, value, merge_with):
+ return value