From 4e9a13142f1ee81c905a2cc9401a88f115ec778e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 20 Jul 2013 12:38:25 -0700 Subject: Init and finalize refactor. Instead of previously initializing and not finalizing the handles that completed successfully when a handler initializing or running failed we should attempt to always give said handlers a chance to finalize (even when another handler fails). --- cloudinit/stages.py | 82 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 29 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index df49cabb..6893afd9 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -383,36 +383,60 @@ class Init(object): # Form our cloud interface data = self.cloudify() - # Init the handlers first - called = [] - for (_ctype, mod) in c_handlers.iteritems(): - if mod in called: - continue - handlers.call_begin(mod, data, frequency) - called.append(mod) - - # Walk the user data - part_data = { - 'handlers': c_handlers, - # Any new handlers that are encountered get writen here - 'handlerdir': idir, - 'data': data, - # The default frequency if handlers don't have one - 'frequency': frequency, - # This will be used when new handlers are found - # to help write there contents to files with numbered - # names... - 'handlercount': 0, - } - handlers.walk(user_data_msg, handlers.walker_callback, data=part_data) + # This list contains the modules initialized (so that we only finalize + # ones that were actually initialized) + inited_handlers = [] + + def init_handlers(): + # Init the handlers first + called = [] + for (_ctype, mod) in c_handlers.iteritems(): + if mod in called: + # Avoid initing the same module twice (if said module + # is registered to more than one content-type). + continue + handlers.call_begin(mod, data, frequency) + inited_handlers.append(mod) + called.append(mod) + + def walk_handlers(): + # Walk the user data + part_data = { + 'handlers': c_handlers, + # Any new handlers that are encountered get writen here + 'handlerdir': idir, + 'data': data, + # The default frequency if handlers don't have one + 'frequency': frequency, + # This will be used when new handlers are found + # to help write there contents to files with numbered + # names... + 'handlercount': 0, + } + handlers.walk(user_data_msg, handlers.walker_callback, + data=part_data) + + def finalize_handlers(): + # Give callbacks opportunity to finalize + called = [] + for (_ctype, mod) in c_handlers.iteritems(): + if mod in called: + # Avoid finalizing the same module twice (if said module + # is registered to more than one content-type). + continue + if mod not in inited_handlers: + continue + called.append(mod) + try: + handlers.call_end(mod, data, frequency) + except: + util.logexc(LOG, "Failed to finalize handler: %s", mod) - # Give callbacks opportunity to finalize - called = [] - for (_ctype, mod) in c_handlers.iteritems(): - if mod in called: - continue - handlers.call_end(mod, data, frequency) - called.append(mod) + try: + init_handlers() + walk_handlers() + finally: + finalize_handlers() # Perform post-consumption adjustments so that # modules that run during the init stage reflect -- cgit v1.2.3 From a3ef9d24c6c913676d22dd7017a1f1b235d47a45 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 20 Jul 2013 12:43:51 -0700 Subject: Update comments + link to bug. LP: #1203368 --- cloudinit/stages.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 6893afd9..ed995628 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -425,6 +425,8 @@ class Init(object): # is registered to more than one content-type). continue if mod not in inited_handlers: + # Said module was never inited in the first place, so lets + # not attempt to finalize those that never got called. continue called.append(mod) try: -- cgit v1.2.3 From 2849c8d3eb44b186e9eaed46080796d56e9529f2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 20 Jul 2013 13:06:55 -0700 Subject: Also handle custom handlers correctly. LP: #1203368 --- cloudinit/handlers/__init__.py | 9 ++++++--- cloudinit/helpers.py | 2 ++ cloudinit/stages.py | 28 +++++++++++----------------- 3 files changed, 19 insertions(+), 20 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 497d68c5..93df5b61 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -151,10 +151,12 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): try: mod = fixup_handler(importer.import_module(modname)) call_begin(mod, pdata['data'], frequency) - # Only register and increment - # after the above have worked (so we don't if it - # fails) + # Only register and increment after the above have worked, so we don't + # register if it fails starting. handlers.register(mod) + # Ensure that it gets finalized by marking said module as having been + # initialized correctly. + handlers.markings[mod].append('initialized') pdata['handlercount'] = curcount + 1 except: util.logexc(LOG, "Failed at registering python file: %s (part " @@ -230,6 +232,7 @@ def walk(msg, callback, data): headers['Content-Type'] = ctype callback(data, filename, part.get_payload(decode=True), headers) partnum = partnum + 1 + return partnum def fixup_handler(mod, def_freq=PER_INSTANCE): diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index b91c1290..bd37b8a3 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -22,6 +22,7 @@ from time import time +import collections import contextlib import io import os @@ -281,6 +282,7 @@ class ContentHandlers(object): def __init__(self): self.registered = {} + self.markings = collections.defaultdict(list) def __contains__(self, item): return self.is_registered(item) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ed995628..43eaca1b 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -383,21 +383,15 @@ class Init(object): # Form our cloud interface data = self.cloudify() - # This list contains the modules initialized (so that we only finalize - # ones that were actually initialized) - inited_handlers = [] - def init_handlers(): # Init the handlers first - called = [] for (_ctype, mod) in c_handlers.iteritems(): - if mod in called: + if 'initialized' in c_handlers.markings[mod]: # Avoid initing the same module twice (if said module # is registered to more than one content-type). continue handlers.call_begin(mod, data, frequency) - inited_handlers.append(mod) - called.append(mod) + c_handlers.markings[mod].append('initialized') def walk_handlers(): # Walk the user data @@ -413,22 +407,22 @@ class Init(object): # names... 'handlercount': 0, } - handlers.walk(user_data_msg, handlers.walker_callback, - data=part_data) + return handlers.walk(user_data_msg, handlers.walker_callback, + data=part_data) def finalize_handlers(): # Give callbacks opportunity to finalize - called = [] for (_ctype, mod) in c_handlers.iteritems(): - if mod in called: - # Avoid finalizing the same module twice (if said module - # is registered to more than one content-type). - continue - if mod not in inited_handlers: + mod_markings = c_handlers.markings[mod] + if 'initialized' not in mod_markings: # Said module was never inited in the first place, so lets # not attempt to finalize those that never got called. continue - called.append(mod) + if 'finalized' in mod_markings: + # Avoid finalizing the same module twice (if said module + # is registered to more than one content-type). + continue + c_handlers.markings[mod].append('finalized') try: handlers.call_end(mod, data, frequency) except: -- cgit v1.2.3 From bbfc76fb74595881b25acc1bbbd426314c2390ed Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 20 Jul 2013 13:13:02 -0700 Subject: Remove return not used. --- cloudinit/handlers/__init__.py | 1 - cloudinit/stages.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 93df5b61..f9b90323 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -232,7 +232,6 @@ def walk(msg, callback, data): headers['Content-Type'] = ctype callback(data, filename, part.get_payload(decode=True), headers) partnum = partnum + 1 - return partnum def fixup_handler(mod, def_freq=PER_INSTANCE): diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 43eaca1b..ba974a3e 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -407,8 +407,8 @@ class Init(object): # names... 'handlercount': 0, } - return handlers.walk(user_data_msg, handlers.walker_callback, - data=part_data) + handlers.walk(user_data_msg, handlers.walker_callback, + data=part_data) def finalize_handlers(): # Give callbacks opportunity to finalize -- cgit v1.2.3 From 971c2b2366c6e58921e1d2dd3ba18e597cbc20e8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 21 Jul 2013 10:45:29 -0700 Subject: Just use an initialized array. --- cloudinit/handlers/__init__.py | 5 +---- cloudinit/helpers.py | 7 ++++--- cloudinit/stages.py | 13 ++++--------- 3 files changed, 9 insertions(+), 16 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index f9b90323..1d450061 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -153,10 +153,7 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): call_begin(mod, pdata['data'], frequency) # Only register and increment after the above have worked, so we don't # register if it fails starting. - handlers.register(mod) - # Ensure that it gets finalized by marking said module as having been - # initialized correctly. - handlers.markings[mod].append('initialized') + handlers.register(mod, initialized=True) pdata['handlercount'] = curcount + 1 except: util.logexc(LOG, "Failed at registering python file: %s (part " diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index bd37b8a3..1c46efde 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -22,7 +22,6 @@ from time import time -import collections import contextlib import io import os @@ -282,7 +281,7 @@ class ContentHandlers(object): def __init__(self): self.registered = {} - self.markings = collections.defaultdict(list) + self.initialized = [] def __contains__(self, item): return self.is_registered(item) @@ -293,11 +292,13 @@ class ContentHandlers(object): def is_registered(self, content_type): return content_type in self.registered - def register(self, mod): + def register(self, mod, initialized=False): types = set() for t in mod.list_types(): self.registered[t] = mod types.add(t) + if initialized and mod not in self.initialized: + self.initialized.append(mod) return types def _get_handler(self, content_type): diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ba974a3e..fade1182 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -386,12 +386,12 @@ class Init(object): def init_handlers(): # Init the handlers first for (_ctype, mod) in c_handlers.iteritems(): - if 'initialized' in c_handlers.markings[mod]: + if mod in c_handlers.initialized: # Avoid initing the same module twice (if said module # is registered to more than one content-type). continue handlers.call_begin(mod, data, frequency) - c_handlers.markings[mod].append('initialized') + c_handlers.initialized.append(mod) def walk_handlers(): # Walk the user data @@ -413,16 +413,11 @@ class Init(object): def finalize_handlers(): # Give callbacks opportunity to finalize for (_ctype, mod) in c_handlers.iteritems(): - mod_markings = c_handlers.markings[mod] - if 'initialized' not in mod_markings: + if mod not in c_handlers.initialized: # Said module was never inited in the first place, so lets # not attempt to finalize those that never got called. continue - if 'finalized' in mod_markings: - # Avoid finalizing the same module twice (if said module - # is registered to more than one content-type). - continue - c_handlers.markings[mod].append('finalized') + c_handlers.initialized.remove(mod) try: handlers.call_end(mod, data, frequency) except: -- cgit v1.2.3 From a5dd2146bb98874219eb449ae06f57203099d4d4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 21 Jul 2013 11:01:55 -0700 Subject: Also make the dir handler registration a simple function. --- cloudinit/stages.py | 57 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 24 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index fade1182..f08589a7 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -344,12 +344,13 @@ class Init(object): cdir = self.paths.get_cpath("handlers") idir = self._get_ipath("handlers") - # Add the path to the plugins dir to the top of our list for import - # instance dir should be read before cloud-dir - if cdir and cdir not in sys.path: - sys.path.insert(0, cdir) - if idir and idir not in sys.path: - sys.path.insert(0, idir) + # Add the path to the plugins dir to the top of our list for importing + # new handlers. + # + # Note(harlowja): instance dir should be read before cloud-dir + for d in [cdir, idir]: + if d and d not in sys.path: + sys.path.insert(0, d) # Ensure datasource fetched before activation (just incase) user_data_msg = self.datasource.get_userdata(True) @@ -357,24 +358,32 @@ class Init(object): # This keeps track of all the active handlers c_handlers = helpers.ContentHandlers() - # Add handlers in cdir - potential_handlers = util.find_modules(cdir) - for (fname, mod_name) in potential_handlers.iteritems(): - try: - mod_locs = importer.find_module(mod_name, [''], - ['list_types', - 'handle_part']) - if not mod_locs: - LOG.warn(("Could not find a valid user-data handler" - " named %s in file %s"), mod_name, fname) - continue - mod = importer.import_module(mod_locs[0]) - mod = handlers.fixup_handler(mod) - types = c_handlers.register(mod) - LOG.debug("Added handler for %s from %s", types, fname) - except: - util.logexc(LOG, "Failed to register handler from %s", fname) - + def register_handlers_in_dir(path): + # Attempts to register any handler modules under the given path. + potential_handlers = util.find_modules(path) + for (fname, mod_name) in potential_handlers.iteritems(): + try: + mod_locs = importer.find_module(mod_name, [''], + ['list_types', + 'handle_part']) + if not mod_locs: + LOG.warn(("Could not find a valid user-data handler" + " named %s in file %s"), mod_name, fname) + continue + mod = importer.import_module(mod_locs[0]) + mod = handlers.fixup_handler(mod) + types = c_handlers.register(mod) + LOG.debug("Added handler for %s from %s", types, fname) + except Exception: + util.logexc(LOG, "Failed to register handler from %s", + fname) + + # Add any handlers in the cloud-dir + register_handlers_in_dir(cdir) + + # Register any other handlers that come from the default set. This + # is done after the cloud-dir handlers so that the cdir modules can + # take over the default user-data handler content-types. def_handlers = self._default_userdata_handlers() applied_def_handlers = c_handlers.register_defaults(def_handlers) if applied_def_handlers: -- cgit v1.2.3 From d655d019fb0a45389d87db39b0ef5001e27e2616 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 21 Jul 2013 11:04:55 -0700 Subject: Ensure what we are searching over is a directory. --- cloudinit/stages.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cloudinit') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index f08589a7..3e49e8c5 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -360,6 +360,8 @@ class Init(object): def register_handlers_in_dir(path): # Attempts to register any handler modules under the given path. + if not path or not os.path.isdir(path): + return potential_handlers = util.find_modules(path) for (fname, mod_name) in potential_handlers.iteritems(): try: -- cgit v1.2.3