From f51e04ba97a42782e6a0e973488290552346387f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 19 Jul 2013 16:32:48 -0700 Subject: Add the ability to merge with jsonpatch. Jsonpatch is a new RFC standard for merging json-like structures which the cloud-init cloud-config is one such structure. To use this in a limited fashion (to start) add the ability for the cloud-config handler to accept this content-type and use it as an alternate way to merge new cloud-config sections into the accumulated cloud-config. LP: #1200476 --- cloudinit/handlers/cloud_config.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index c97ca3e8..8d1ba37f 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -20,6 +20,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import jsonpatch + from cloudinit import handlers from cloudinit import log as logging from cloudinit import mergers @@ -50,6 +52,9 @@ MERGE_HEADER = 'Merge-Type' # This gets loaded into yaml with final result {'a': 22} DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()') +# See: https://tools.ietf.org/html/rfc6902 +JSON_PATCH_CTYPE = 'application/json-patch+json' + class CloudConfigPartHandler(handlers.Handler): def __init__(self, paths, **_kwargs): @@ -59,9 +64,11 @@ class CloudConfigPartHandler(handlers.Handler): self.file_names = [] def list_types(self): - return [ + ctypes_handled = [ handlers.type_from_starts_with("#cloud-config"), + JSON_PATCH_CTYPE, ] + return ctypes_handled def _write_cloud_config(self): if not self.cloud_fn: @@ -107,13 +114,15 @@ class CloudConfigPartHandler(handlers.Handler): all_mergers = DEF_MERGERS return (payload_yaml, all_mergers) + def _merge_patch(self, payload): + patch = jsonpatch.JsonPatch.from_string(payload) + LOG.debug("Merging by applying json patch %s", patch) + self.cloud_buf = patch.apply(self.cloud_buf, in_place=False) + def _merge_part(self, payload, headers): (payload_yaml, my_mergers) = self._extract_mergers(payload, headers) LOG.debug("Merging by applying %s", my_mergers) merger = mergers.construct(my_mergers) - if self.cloud_buf is None: - # First time through, merge with an empty dict... - self.cloud_buf = {} self.cloud_buf = merger.merge(self.cloud_buf, payload_yaml) def _reset(self): @@ -130,7 +139,13 @@ class CloudConfigPartHandler(handlers.Handler): self._reset() return try: - self._merge_part(payload, headers) + # First time through, merge with an empty dict... + if self.cloud_buf is None or not self.file_names: + self.cloud_buf = {} + if ctype == JSON_PATCH_CTYPE: + self._merge_patch(payload) + else: + self._merge_part(payload, headers) # Ensure filename is ok to store for i in ("\n", "\r", "\t"): filename = filename.replace(i, " ") -- cgit v1.2.3 From 36bbd898e9b8bef508b5d185dc1e52af0f13cfd0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 20 Jul 2013 05:56:30 -0700 Subject: Add usage of '#json-patch' --- cloudinit/handlers/__init__.py | 1 + cloudinit/handlers/cloud_config.py | 17 ++++++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 497d68c5..297e7451 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -62,6 +62,7 @@ INCLUSION_TYPES_MAP = { '#part-handler': 'text/part-handler', '#cloud-boothook': 'text/cloud-boothook', '#cloud-config-archive': 'text/cloud-config-archive', + '#json-patch': 'application/json-patch+json', } # Sorted longest first diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 8d1ba37f..84653375 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -66,22 +66,21 @@ class CloudConfigPartHandler(handlers.Handler): def list_types(self): ctypes_handled = [ handlers.type_from_starts_with("#cloud-config"), - JSON_PATCH_CTYPE, + handlers.type_from_starts_with("#json-patch"), ] return ctypes_handled def _write_cloud_config(self): - if not self.cloud_fn: + if not self.cloud_fn or not len(self.file_names): return # Capture which files we merged from... file_lines = [] - if self.file_names: - file_lines.append("# from %s files" % (len(self.file_names))) - for fn in self.file_names: - if not fn: - fn = '?' - file_lines.append("# %s" % (fn)) - file_lines.append("") + file_lines.append("# from %s files" % (len(self.file_names))) + for fn in self.file_names: + if not fn: + fn = '?' + file_lines.append("# %s" % (fn)) + file_lines.append("") if self.cloud_buf is not None: # Something was actually gathered.... lines = [ -- cgit v1.2.3 From 03d93b4fe94d0cd4de8fb661748207ea251fbb2a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 20 Jul 2013 06:26:30 -0700 Subject: Fix content-type constant. --- cloudinit/handlers/cloud_config.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 84653375..3a4f2150 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -52,9 +52,6 @@ MERGE_HEADER = 'Merge-Type' # This gets loaded into yaml with final result {'a': 22} DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()') -# See: https://tools.ietf.org/html/rfc6902 -JSON_PATCH_CTYPE = 'application/json-patch+json' - class CloudConfigPartHandler(handlers.Handler): def __init__(self, paths, **_kwargs): -- cgit v1.2.3 From 1f73904df31feb22ad8545a3a336ba2e92e367bb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 20 Jul 2013 07:01:33 -0700 Subject: Fix constant move. --- cloudinit/handlers/cloud_config.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 3a4f2150..99dc71d0 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -52,6 +52,12 @@ MERGE_HEADER = 'Merge-Type' # This gets loaded into yaml with final result {'a': 22} DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()') +# The file header -> content types this module will handle. +CC_TYPES = { + '#json-patch': handlers.type_from_starts_with("#json-patch"), + '#cloud-config': handlers.type_from_starts_with("#cloud-config"), +} + class CloudConfigPartHandler(handlers.Handler): def __init__(self, paths, **_kwargs): @@ -61,23 +67,20 @@ class CloudConfigPartHandler(handlers.Handler): self.file_names = [] def list_types(self): - ctypes_handled = [ - handlers.type_from_starts_with("#cloud-config"), - handlers.type_from_starts_with("#json-patch"), - ] - return ctypes_handled + return list(CC_TYPES.values()) def _write_cloud_config(self): - if not self.cloud_fn or not len(self.file_names): + if not self.cloud_fn: return # Capture which files we merged from... file_lines = [] - file_lines.append("# from %s files" % (len(self.file_names))) - for fn in self.file_names: - if not fn: - fn = '?' - file_lines.append("# %s" % (fn)) - file_lines.append("") + if self.file_names: + file_lines.append("# from %s files" % (len(self.file_names))) + for fn in self.file_names: + if not fn: + fn = '?' + file_lines.append("# %s" % (fn)) + file_lines.append("") if self.cloud_buf is not None: # Something was actually gathered.... lines = [ @@ -138,7 +141,7 @@ class CloudConfigPartHandler(handlers.Handler): # First time through, merge with an empty dict... if self.cloud_buf is None or not self.file_names: self.cloud_buf = {} - if ctype == JSON_PATCH_CTYPE: + if ctype == CC_TYPES['#json-patch']: self._merge_patch(payload) else: self._merge_part(payload, headers) -- cgit v1.2.3 From 0be217f4c177a34b5ec46aa3e64fb1bede4ceb33 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 20 Jul 2013 07:34:31 -0700 Subject: Remove json-patch inclusion header if payload contains it. --- cloudinit/handlers/cloud_config.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 99dc71d0..654314c0 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -114,6 +114,12 @@ class CloudConfigPartHandler(handlers.Handler): return (payload_yaml, all_mergers) def _merge_patch(self, payload): + if payload.startswith("#json-patch"): + # JSON doesn't handle comments in this manner, so ensure that + # if we started with this 'type' that we remove it before + # attempting to load it as json (which the jsonpatch library will + # attempt to do). + payload = payload[len("#json-patch"):] patch = jsonpatch.JsonPatch.from_string(payload) LOG.debug("Merging by applying json patch %s", patch) self.cloud_buf = patch.apply(self.cloud_buf, in_place=False) -- cgit v1.2.3 From eae3b6ad499b88b725a52cf07245e4721af380cf Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 20 Jul 2013 07:37:02 -0700 Subject: Ensure we remove the same way we detect. --- cloudinit/handlers/cloud_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 654314c0..4dcdbe8b 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -114,7 +114,8 @@ class CloudConfigPartHandler(handlers.Handler): return (payload_yaml, all_mergers) def _merge_patch(self, payload): - if payload.startswith("#json-patch"): + payload = payload.lstrip() + if payload.lower().startswith("#json-patch"): # JSON doesn't handle comments in this manner, so ensure that # if we started with this 'type' that we remove it before # attempting to load it as json (which the jsonpatch library will -- cgit v1.2.3 From 243df010c49de52be0ca9159e15378bb335b1163 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 24 Jul 2013 11:04:56 -0400 Subject: change 'json-patch' to 'cloud-config-jsonp' --- cloudinit/handlers/__init__.py | 2 +- cloudinit/handlers/cloud_config.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 4c7c9295..2ddc75f4 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -62,7 +62,7 @@ INCLUSION_TYPES_MAP = { '#part-handler': 'text/part-handler', '#cloud-boothook': 'text/cloud-boothook', '#cloud-config-archive': 'text/cloud-config-archive', - '#json-patch': 'application/json-patch+json', + '#cloud-config-jsonp': 'text/cloud-config-jsonp', } # Sorted longest first diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 0f080e66..8bbc904d 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -54,8 +54,9 @@ DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()') CLOUD_PREFIX = "#cloud-config" # The file header -> content types this module will handle. +CC_JSONP_PRE = "#cloud-config-jsonp" CC_TYPES = { - '#json-patch': handlers.type_from_starts_with("#json-patch"), + CC_JSONP_PRE: handlers.type_from_starts_with(CC_JSONP_PRE), '#cloud-config': handlers.type_from_starts_with("#cloud-config"), } @@ -116,12 +117,12 @@ class CloudConfigPartHandler(handlers.Handler): def _merge_patch(self, payload): payload = payload.lstrip() - if payload.lower().startswith("#json-patch"): + if payload.lower().startswith(CC_JSONP_PRE): # JSON doesn't handle comments in this manner, so ensure that # if we started with this 'type' that we remove it before # attempting to load it as json (which the jsonpatch library will # attempt to do). - payload = payload[len("#json-patch"):] + payload = payload[CC_JSONP_PRE:] patch = jsonpatch.JsonPatch.from_string(payload) LOG.debug("Merging by applying json patch %s", patch) self.cloud_buf = patch.apply(self.cloud_buf, in_place=False) @@ -149,7 +150,7 @@ class CloudConfigPartHandler(handlers.Handler): # First time through, merge with an empty dict... if self.cloud_buf is None or not self.file_names: self.cloud_buf = {} - if ctype == CC_TYPES['#json-patch']: + if ctype == CC_TYPES[CC_JSONP_PRE]: self._merge_patch(payload) else: self._merge_part(payload, headers) -- cgit v1.2.3 From 97b19f3b1992e56ef8e1a055fbe64a19d1eacfbf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 24 Jul 2013 11:07:55 -0400 Subject: rename CC_JSONP_PRE again (JSONP_PREFIX) and use CLOUD_PREFIX --- cloudinit/handlers/cloud_config.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 8bbc904d..7edae13d 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -52,12 +52,12 @@ MERGE_HEADER = 'Merge-Type' # This gets loaded into yaml with final result {'a': 22} DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()') CLOUD_PREFIX = "#cloud-config" +JSONP_PREFIX = "#cloud-config-jsonp" # The file header -> content types this module will handle. -CC_JSONP_PRE = "#cloud-config-jsonp" CC_TYPES = { - CC_JSONP_PRE: handlers.type_from_starts_with(CC_JSONP_PRE), - '#cloud-config': handlers.type_from_starts_with("#cloud-config"), + JSONP_PREFIX: handlers.type_from_starts_with(JSONP_PREFIX), + CLOUD_PREFIX: handlers.type_from_starts_with(CLOUD_PREFIX), } @@ -117,12 +117,12 @@ class CloudConfigPartHandler(handlers.Handler): def _merge_patch(self, payload): payload = payload.lstrip() - if payload.lower().startswith(CC_JSONP_PRE): + if payload.lower().startswith(JSONP_PREFIX): # JSON doesn't handle comments in this manner, so ensure that # if we started with this 'type' that we remove it before # attempting to load it as json (which the jsonpatch library will # attempt to do). - payload = payload[CC_JSONP_PRE:] + payload = payload[JSONP_PREFIX:] patch = jsonpatch.JsonPatch.from_string(payload) LOG.debug("Merging by applying json patch %s", patch) self.cloud_buf = patch.apply(self.cloud_buf, in_place=False) @@ -150,7 +150,7 @@ class CloudConfigPartHandler(handlers.Handler): # First time through, merge with an empty dict... if self.cloud_buf is None or not self.file_names: self.cloud_buf = {} - if ctype == CC_TYPES[CC_JSONP_PRE]: + if ctype == CC_TYPES[JSONP_PREFIX]: self._merge_patch(payload) else: self._merge_part(payload, headers) -- cgit v1.2.3