diff options
| -rw-r--r-- | doc/merging.rst (renamed from doc/merging.txt) | 137 | ||||
| -rw-r--r-- | doc/rtd/index.rst | 1 | ||||
| -rw-r--r-- | doc/rtd/topics/merging.rst | 5 | ||||
| -rw-r--r-- | tests/unittests/helpers.py | 1 | ||||
| -rw-r--r-- | tests/unittests/test_builtin_handlers.py | 24 |
5 files changed, 94 insertions, 74 deletions
diff --git a/doc/merging.txt b/doc/merging.rst index f719aec8..6344facd 100644 --- a/doc/merging.txt +++ b/doc/merging.rst @@ -1,6 +1,3 @@ -Arriving in 0.7.2 is a new way to handle dictionary merging in cloud-init. ---- - Overview -------- @@ -16,32 +13,38 @@ to determine exactly how there objects will be merged. For example. -#cloud-config (1) -run_cmd: - - bash1 - - bash2 +.. code-block:: yaml -#cloud-config (2) -run_cmd: - - bash3 - - bash4 + #cloud-config (1) + run_cmd: + - bash1 + - bash2 + + #cloud-config (2) + run_cmd: + - bash3 + - bash4 The previous way of merging the following 2 objects would result in a final cloud-config object that contains the following. -#cloud-config (merged) -run_cmd: - - bash3 - - bash4 +.. code-block:: yaml + + #cloud-config (merged) + run_cmd: + - bash3 + - bash4 Typically this is not what users want, instead they would likely prefer: -#cloud-config (merged) -run_cmd: - - bash1 - - bash2 - - bash3 - - bash4 +.. code-block:: yaml + + #cloud-config (merged) + run_cmd: + - bash1 + - bash2 + - bash3 + - bash4 This way makes it easier to combine the various cloud-config objects you have into a more useful list, thus reducing duplication that would have had to @@ -51,7 +54,7 @@ Customizability --------------- Since the above merging algorithm may not always be the desired merging -algorithm (like how the merging algorithm in < 0.7.2 was not always the preferred +algorithm (like how the previous merging algorithm was not always the preferred one) the concept of customizing how merging can be done was introduced through a new concept call 'merge classes'. @@ -60,42 +63,44 @@ to merge a given type with another given type. An example of one of these merging classes is the following: -class Merger(object): - def __init__(self, merger, opts): - self._merger = merger - self._overwrite = 'overwrite' in opts - - # This merging algorithm will attempt to merge with - # another dictionary, on encountering any other type of object - # it will not merge with said object, but will instead return - # the original value - # - # On encountering a dictionary, it will create a new dictionary - # composed of the original and the one to merge with, if 'overwrite' - # is enabled then keys that exist in the original will be overwritten - # by keys in the one to merge with (and associated values). Otherwise - # if not in overwrite mode the 2 conflicting keys themselves will - # be merged. - def _on_dict(self, value, merge_with): - if not isinstance(merge_with, (dict)): - return value - merged = dict(value) - for (k, v) in merge_with.items(): - if k in merged: - if not self._overwrite: - merged[k] = self._merger.merge(merged[k], v) - else: - merged[k] = v - else: - merged[k] = v - return merged +.. code-block:: python + + class Merger(object): + def __init__(self, merger, opts): + self._merger = merger + self._overwrite = 'overwrite' in opts + + # This merging algorithm will attempt to merge with + # another dictionary, on encountering any other type of object + # it will not merge with said object, but will instead return + # the original value + # + # On encountering a dictionary, it will create a new dictionary + # composed of the original and the one to merge with, if 'overwrite' + # is enabled then keys that exist in the original will be overwritten + # by keys in the one to merge with (and associated values). Otherwise + # if not in overwrite mode the 2 conflicting keys themselves will + # be merged. + def _on_dict(self, value, merge_with): + if not isinstance(merge_with, (dict)): + return value + merged = dict(value) + for (k, v) in merge_with.items(): + if k in merged: + if not self._overwrite: + merged[k] = self._merger.merge(merged[k], v) + else: + merged[k] = v + else: + merged[k] = v + return merged As you can see there is a '_on_dict' method here that will be given a source value and a value to merge with. The result will be the merged object. This code itself is called by another merging class which 'directs' the merging to happen by analyzing the types of the objects to merge and attempting to find a know object that will merge that type. I will avoid pasting that here, but it can be found -in the mergers/__init__.py file (see LookupMerger and UnknownMerger). +in the `mergers/__init__.py` file (see `LookupMerger` and `UnknownMerger`). So following the typical cloud-init way of allowing source code to be downloaded and used dynamically, it is possible for users to inject there own merging files @@ -123,11 +128,14 @@ for your own usage. or as a dictionary (see format below). The keys that are looked up for this definition are the following (in order), 'merge_how', 'merge_type'. -*String format* +String format +******** The string format that is expected is the following. -"classname(option1,option2)+classname2(option3,option4)" (and so on) +:: + + classname1(option1,option2)+classname2(option3,option4).... The class name there will be connected to class names used when looking for the class that can be used to merge and options provided will be given to the class @@ -135,20 +143,21 @@ on construction of that class. For example, the default string that is used when none is provided is the following: -"list(extend)+dict()+str(append)" +:: + + list(extend)+dict()+str(append) -*Dictionary format* +Dictionary format +******** In cases where a dictionary can be used to specify the same information as the string format (ie option #2 of above) it can be used, for example. -merge_how: - - name: list - settings: [extend] - - name: dict - settings: [] - - name: str - settings: [append] +.. code-block:: python + + {'merge_how': [{'name': 'list', 'settings': ['extend']}, + {'name': 'dict', 'settings': []}, + {'name': 'str', 'settings': ['append']}]} This would be the equivalent format for default string format but in dictionary form instead of string form. @@ -172,7 +181,7 @@ cloud-config dictionary coming after it. Other uses ---------- -The default merging algorithm for merging conf.d yaml files (which form a initial +The default merging algorithm for merging 'conf.d' yaml files (which form a initial yaml config for cloud-init) was also changed to use this mechanism so its full benefits (and customization) can also be used there as well. Other places that used the previous merging are also similar now extensible (metadata merging for diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst index 619bb5dc..fe04b1a9 100644 --- a/doc/rtd/index.rst +++ b/doc/rtd/index.rst @@ -24,6 +24,7 @@ Summary topics/examples topics/datasources topics/modules + topics/merging topics/moreinfo topics/hacking diff --git a/doc/rtd/topics/merging.rst b/doc/rtd/topics/merging.rst new file mode 100644 index 00000000..8a03f3c7 --- /dev/null +++ b/doc/rtd/topics/merging.rst @@ -0,0 +1,5 @@ +========= +Merging +========= + +.. include:: ../../merging.rst diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 904677f1..e020a3ec 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -175,6 +175,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): def patchOS(self, new_root): patch_funcs = { os.path: ['isfile', 'exists', 'islink', 'isdir'], + os: ['listdir'], } for (mod, funcs) in patch_funcs.items(): for f in funcs: diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index da52f15b..dace486a 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -1,9 +1,8 @@ """Tests of the built-in user data handlers.""" import os -import unittest -from mocker import MockerTestCase +from tests.unittests import helpers as test_helpers from cloudinit import handlers from cloudinit import helpers @@ -14,7 +13,7 @@ from cloudinit.handlers import upstart_job from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) -class TestBuiltins(MockerTestCase): +class TestBuiltins(test_helpers.FilesystemMockingTestCase): def test_upstart_frequency_no_out(self): c_root = self.makeDir() @@ -35,16 +34,20 @@ class TestBuiltins(MockerTestCase): None, None, None) self.assertEquals(0, len(os.listdir(up_root))) - @unittest.skip("until LP: #1124384 fixed") def test_upstart_frequency_single(self): # files should be written out when frequency is ! per-instance - c_root = self.makeDir() - up_root = self.makeDir() + new_root = self.makeDir() + freq = PER_INSTANCE + + self.patchOS(new_root) + self.patchUtils(new_root) paths = helpers.Paths({ - 'cloud_dir': c_root, - 'upstart_dir': up_root, + 'upstart_dir': "/etc/upstart", }) - freq = PER_INSTANCE + + util.ensure_dir("/run") + util.ensure_dir("/etc/upstart") + util.write_file("/run/cloud-init-upstart-reload", 'test') mock_subp = self.mocker.replace(util.subp, passthrough=False) mock_subp(["initctl", "reload-configuration"], capture=False) @@ -57,4 +60,5 @@ class TestBuiltins(MockerTestCase): 'test.conf', 'blah', freq) h.handle_part('', handlers.CONTENT_END, None, None, None) - self.assertEquals(1, len(os.listdir(up_root))) + + self.assertEquals(1, len(os.listdir('/etc/upstart'))) |