From 8bc85abd97e06d964bbd26208eb732e80eb87c10 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 20 Nov 2012 20:02:48 -0800 Subject: Start allowing different merging types to be applied After user data handling splits apart all the different content types into there various mime messages it is nice to be able to have each message specify how it should be merged (mainly for cloud-config or cloud-archive) into the single cloud config that is eventually used. This starts to add a plugable merging framework and the needed components to activate said headers and merging. --- cloudinit/handlers/upstart_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/handlers/upstart_job.py') diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index 99e0afde..a5cb9b0c 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -42,7 +42,7 @@ class UpstartJobPartHandler(handlers.Handler): handlers.type_from_starts_with("#upstart-job"), ] - def _handle_part(self, _data, ctype, filename, payload, frequency): + def handle_part(self, _data, ctype, filename, payload, frequency): if ctype in handlers.CONTENT_SIGNALS: return -- cgit v1.2.3 From 6a72a8677ad2e4e66669d2be93880643b0525b51 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 27 Feb 2013 13:02:06 -0500 Subject: do not reload upstart configuration on upstart jobs For now, we disable reloading upstart jobs due to bug 1124384. At some point in the future, we could enable it again when that bug is fixed. The change here allows for a boothook in a multipart input to write the file '/run/cloud-init-upstart-reload' and then have configuration reloaded. --- cloudinit/handlers/upstart_job.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'cloudinit/handlers/upstart_job.py') diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index 4684f7f2..0aa7446e 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -65,6 +65,14 @@ class UpstartJobPartHandler(handlers.Handler): path = os.path.join(self.upstart_dir, filename) util.write_file(path, payload, 0644) - # if inotify support is not present in the root filesystem - # (overlayroot) then we need to tell upstart to re-read /etc - util.subp(["initctl", "reload-configuration"], capture=False) + # FIXME LATER (LP: #1124384) + # a bug in upstart means that invoking reload-configuration + # at this stage in boot causes havoc. So, until that is fixed + # we will not do that. However, I'd like to be able to easily + # test to see if this bug is still present in an image with + # a newer upstart. So, a boot hook could easiliy write this file. + if os.path.exists("/run/cloud-init-upstart-reload"): + # if inotify support is not present in the root filesystem + # (overlayroot) then we need to tell upstart to re-read /etc + + util.subp(["initctl", "reload-configuration"], capture=False) -- cgit v1.2.3 From 5da3984c2ca9e94b2483ab89ecdb5c93b5afb9f8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 7 Mar 2013 17:13:05 -0500 Subject: more pep8/pylint. all clean now --- cloudinit/handlers/boot_hook.py | 3 ++- cloudinit/handlers/cloud_config.py | 7 ++++--- cloudinit/handlers/shell_script.py | 3 ++- cloudinit/handlers/upstart_job.py | 3 ++- cloudinit/mergers/__init__.py | 2 -- tests/unittests/test__init__.py | 3 ++- tests/unittests/test_merging.py | 1 - tests/unittests/test_userdata.py | 6 +++--- 8 files changed, 15 insertions(+), 13 deletions(-) (limited to 'cloudinit/handlers/upstart_job.py') diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index bf313f10..bf2899ab 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -56,7 +56,8 @@ class BootHookPartHandler(handlers.Handler): util.write_file(filepath, contents, 0700) return filepath - def handle_part(self, _data, ctype, filename, payload, _frequency): + def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 + payload, frequency): # pylint: disable=W0613 if ctype in handlers.CONTENT_SIGNALS: return diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 5f519f78..d30d6338 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -46,7 +46,7 @@ class CloudConfigPartHandler(handlers.Handler): handlers.type_from_starts_with("#cloud-config"), ] - def _write_cloud_config(self, buf): + def _write_cloud_config(self): if not self.cloud_fn: return # Capture which files we merged from... @@ -107,12 +107,13 @@ class CloudConfigPartHandler(handlers.Handler): self.cloud_buf = None self.mergers = [DEF_MERGERS] - def handle_part(self, _data, ctype, filename, payload, _freq, headers): + def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 + payload, _frequency, headers): # pylint: disable=W0613 if ctype == handlers.CONTENT_START: self._reset() return if ctype == handlers.CONTENT_END: - self._write_cloud_config(self.cloud_buf) + self._write_cloud_config() self._reset() return try: diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py index 2a87e8dd..b185c374 100644 --- a/cloudinit/handlers/shell_script.py +++ b/cloudinit/handlers/shell_script.py @@ -41,7 +41,8 @@ class ShellScriptPartHandler(handlers.Handler): handlers.type_from_starts_with("#!"), ] - def handle_part(self, _data, ctype, filename, payload, _frequency): + def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 + payload, frequency): # pylint: disable=W0613 if ctype in handlers.CONTENT_SIGNALS: # TODO(harlowja): maybe delete existing things here return diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index 3d8833a1..edd56527 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -42,7 +42,8 @@ class UpstartJobPartHandler(handlers.Handler): handlers.type_from_starts_with("#upstart-job"), ] - def handle_part(self, _data, ctype, filename, payload, frequency): + def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 + payload, frequency): if ctype in handlers.CONTENT_SIGNALS: return diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index ac16f143..e1ff57ba 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -152,5 +152,3 @@ def construct(parsed_mergers): for (attr, opts) in mergers_to_be: mergers.append(attr(root, opts)) return root - - diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 56ccbcfb..2c0abfbc 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -22,7 +22,8 @@ class FakeModule(handlers.Handler): def list_types(self): return self.types - def handle_part(self, data, ctype, filename, payload, frequency): + def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 + payload, frequency): pass diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 591a99c8..ad137e85 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -140,4 +140,3 @@ class TestSimpleRun(helpers.MockerTestCase): 'e': 'f', } }) - diff --git a/tests/unittests/test_userdata.py b/tests/unittests/test_userdata.py index 48ad9c5f..fdfe2542 100644 --- a/tests/unittests/test_userdata.py +++ b/tests/unittests/test_userdata.py @@ -55,7 +55,7 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): #cloud-config a: b e: f -run: +run: - b - c ''' @@ -67,7 +67,7 @@ run: #cloud-config a: e e: g -run: +run: - stuff - morestuff ''' @@ -77,7 +77,7 @@ run: blob3 = ''' #cloud-config -e: +e: - 1 - 2 - 3 -- cgit v1.2.3 From 6cb13e582c66068b26231d31c5db6a987fc4a2e6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 02:26:28 -0400 Subject: handlers/upstart_job.py: invoke reload-configuration if we can. Previously we'd not invoked reload-configuration because doing so caused issues with upstart. Now, instead, determine if we can invoke it. LP: #1124384 --- ChangeLog | 2 ++ cloudinit/handlers/upstart_job.py | 60 ++++++++++++++++++++++++++++++++------- 2 files changed, 52 insertions(+), 10 deletions(-) (limited to 'cloudinit/handlers/upstart_job.py') diff --git a/ChangeLog b/ChangeLog index 6495a19b..40c8912a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,8 @@ 0.7.3: - fix omnibus chef installer (LP: #1182265) [Chris Wing] - small fix for OVF datasource for iso transport on non-iso9660 filesystem + - determine if upstart version is suitable for + 'initctl reload-configuration' (LP: #1124384). If so, then invoke it. 0.7.2: - add a debian watch file - add 'sudo' entry to ubuntu's default user (LP: #1080717) diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index edd56527..b3d60d6e 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -22,6 +22,7 @@ import os +import re from cloudinit import handlers from cloudinit import log as logging @@ -66,14 +67,53 @@ class UpstartJobPartHandler(handlers.Handler): path = os.path.join(self.upstart_dir, filename) util.write_file(path, payload, 0644) - # FIXME LATER (LP: #1124384) - # a bug in upstart means that invoking reload-configuration - # at this stage in boot causes havoc. So, until that is fixed - # we will not do that. However, I'd like to be able to easily - # test to see if this bug is still present in an image with - # a newer upstart. So, a boot hook could easiliy write this file. - if os.path.exists("/run/cloud-init-upstart-reload"): - # if inotify support is not present in the root filesystem - # (overlayroot) then we need to tell upstart to re-read /etc - + if SUITABLE_UPSTART: util.subp(["initctl", "reload-configuration"], capture=False) + + +def _has_suitable_upstart(): + # (LP: #1124384) + # a bug in upstart means that invoking reload-configuration + # at this stage in boot causes havoc. So, try to determine if upstart + # is installed, and reloading configuration is OK. + if not os.path.exists("/sbin/initctl"): + return False + try: + (version_out, _err) = util.subp(["initctl", "version"]) + except: + util.logexc(LOG, "initctl version failed") + return False + + # expecting 'initctl version' to output something like: init (upstart X.Y) + if re.match("upstart 1.[0-7][\)]", version_out): + return False + if "upstart 0." in version_out: + return False + elif "upstart 1.8" in version_out: + if not os.path.exists("/usr/bin/dpkg-query"): + return False + try: + (dpkg_ver, _err) = util.subp(["dpkg-query", + "--showformat=${Version}", + "--show", "upstart"], rcs=[0, 1]) + except Exception: + util.logexc(LOG, "dpkg-query failed") + return False + + try: + util.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good]) + print "good version" + return True + except util.ProcessExecutionError as e: + if e.exit_code is 1: + pass + else: + util.logexc(LOG, "dpkg --compare-versions failed [%s]", + e.exit_code) + except Exception as e: + util.logexc(LOG, "dpkg --compare-versions failed") + return False + else: + return True + +SUITABLE_UPSTART = _has_suitable_upstart() -- cgit v1.2.3 From 078f0fc644e640eacad316aeda7ee05eb8957aa3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 11 Jul 2013 10:14:49 -0400 Subject: upstart_job.py: fix pylint and bad bad variable in _has_suitable_upstart --- cloudinit/handlers/upstart_job.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/handlers/upstart_job.py') diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index b3d60d6e..7a73d1b2 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -85,7 +85,7 @@ def _has_suitable_upstart(): return False # expecting 'initctl version' to output something like: init (upstart X.Y) - if re.match("upstart 1.[0-7][\)]", version_out): + if re.match("upstart 1.[0-7][)]", version_out): return False if "upstart 0." in version_out: return False @@ -101,8 +101,8 @@ def _has_suitable_upstart(): return False try: + good = "1.8-0ubuntu1.2" util.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good]) - print "good version" return True except util.ProcessExecutionError as e: if e.exit_code is 1: -- cgit v1.2.3 From 7022512f3ceb955be2834844f05d4683f78ff276 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 21 Jul 2013 09:26:44 -0700 Subject: Use constants for repeated type strings. --- cloudinit/handlers/boot_hook.py | 13 ++++++------- cloudinit/handlers/cloud_config.py | 5 +++-- cloudinit/handlers/shell_script.py | 3 ++- cloudinit/handlers/upstart_job.py | 3 ++- 4 files changed, 13 insertions(+), 11 deletions(-) (limited to 'cloudinit/handlers/upstart_job.py') diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index 11ac4fe5..5e7b6204 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -29,6 +29,7 @@ from cloudinit import util from cloudinit.settings import (PER_ALWAYS) LOG = logging.getLogger(__name__) +BOOTHOOK_PREFIX = "#cloud-boothook" class BootHookPartHandler(handlers.Handler): @@ -41,18 +42,16 @@ class BootHookPartHandler(handlers.Handler): def list_types(self): return [ - handlers.type_from_starts_with("#cloud-boothook"), + handlers.type_from_starts_with(BOOTHOOK_PREFIX), ] def _write_part(self, payload, filename): filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - prefix = "#cloud-boothook" - start = 0 - if payload.startswith(prefix): - start = len(prefix) + 1 filepath = os.path.join(self.boothook_dir, filename) - contents = payload[start:] + contents = util.dos2unix(payload) + if contents.startswith(BOOTHOOK_PREFIX): + real_start = len(BOOTHOOK_PREFIX) + 1 + contents = contents[real_start:] util.write_file(filepath, contents, 0700) return filepath diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index c97ca3e8..730672d7 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -49,6 +49,7 @@ MERGE_HEADER = 'Merge-Type' # # This gets loaded into yaml with final result {'a': 22} DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()') +CLOUD_PREFIX = "#cloud-config" class CloudConfigPartHandler(handlers.Handler): @@ -60,7 +61,7 @@ class CloudConfigPartHandler(handlers.Handler): def list_types(self): return [ - handlers.type_from_starts_with("#cloud-config"), + handlers.type_from_starts_with(CLOUD_PREFIX), ] def _write_cloud_config(self): @@ -78,7 +79,7 @@ class CloudConfigPartHandler(handlers.Handler): if self.cloud_buf is not None: # Something was actually gathered.... lines = [ - "#cloud-config", + CLOUD_PREFIX, '', ] lines.extend(file_lines) diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py index b185c374..62289d98 100644 --- a/cloudinit/handlers/shell_script.py +++ b/cloudinit/handlers/shell_script.py @@ -29,6 +29,7 @@ from cloudinit import util from cloudinit.settings import (PER_ALWAYS) LOG = logging.getLogger(__name__) +SHELL_PREFIX = "#!" class ShellScriptPartHandler(handlers.Handler): @@ -38,7 +39,7 @@ class ShellScriptPartHandler(handlers.Handler): def list_types(self): return [ - handlers.type_from_starts_with("#!"), + handlers.type_from_starts_with(SHELL_PREFIX), ] def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index 7a73d1b2..bac4cad2 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -31,6 +31,7 @@ from cloudinit import util from cloudinit.settings import (PER_INSTANCE) LOG = logging.getLogger(__name__) +UPSTART_PREFIX = "#upstart-job" class UpstartJobPartHandler(handlers.Handler): @@ -40,7 +41,7 @@ class UpstartJobPartHandler(handlers.Handler): def list_types(self): return [ - handlers.type_from_starts_with("#upstart-job"), + handlers.type_from_starts_with(UPSTART_PREFIX), ] def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 -- cgit v1.2.3