From 262e54bd5eb4d35e021b2401995eac8ea8d06bb2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 19:11:14 -0700 Subject: Move to having a parts directory/module + seperate modules. --- cloudinit/cloud.py | 37 +++---- cloudinit/parts.py | 214 ---------------------------------------- cloudinit/parts/__init__.py | 117 ++++++++++++++++++++++ cloudinit/parts/boot_hook.py | 42 ++++++++ cloudinit/parts/cloud_config.py | 36 +++++++ cloudinit/parts/shell_script.py | 27 +++++ cloudinit/parts/upstart_job.py | 30 ++++++ 7 files changed, 272 insertions(+), 231 deletions(-) delete mode 100644 cloudinit/parts.py create mode 100644 cloudinit/parts/__init__.py create mode 100644 cloudinit/parts/boot_hook.py create mode 100644 cloudinit/parts/cloud_config.py create mode 100644 cloudinit/parts/shell_script.py create mode 100644 cloudinit/parts/upstart_job.py diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index a98bfbbd..f9c0d531 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -2,22 +2,27 @@ from time import time import cPickle as pickle import contextlib +import copy import os import sys import weakref - from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, OLD_CLOUD_CONFIG, CLOUD_CONFIG, - CFG_BUILTIN, CUR_INSTANCE_LINK) + CUR_INSTANCE_LINK) from cloudinit import (get_builtin_cfg, get_base_cfg) from cloudinit import log as logging -from cloudinit import parts from cloudinit import sources from cloudinit import util from cloudinit import user_data from cloudinit import handlers +from cloudinit import parts +from cloudinit.parts import boot_hook as bh_part +from cloudinit.parts import cloud_config as cc_part +from cloudinit.parts import upstart_job as up_part +from cloudinit.parts import shell_script as ss_part + LOG = logging.getLogger(__name__) @@ -128,10 +133,10 @@ class CloudPaths(object): return ipath -class CloudPartData(object): - def __init__(self, datasource, paths): - self.datasource = datasource - self.paths = paths +class CloudSimple(object): + def __init__(self, init): + self.datasource = init.datasource + self.paths = init.paths def get_userdata(self): return self.datasource.get_userdata() @@ -288,7 +293,7 @@ class CloudInit(object): sys.path.insert(0, idir) # Data will be a little proxy that modules can use - data = CloudPartData(self.datasource, self.paths) + data = CloudSimple(self) # This keeps track of all the active handlers handlers = CloudHandlers(self) @@ -369,13 +374,13 @@ class CloudHandlers(object): def _get_default_handlers(self): def_handlers = [] if self.paths.get_ipath("cloud_config"): - def_handlers.append(parts.CloudConfigPartHandler(self.paths.get_ipath("cloud_config"))) + def_handlers.append(cc_part.CloudConfigPartHandler(self.paths.get_ipath("cloud_config"))) if self.paths.get_ipath_cur('scripts'): - def_handlers.append(parts.ShellScriptPartHandler(self.paths.get_ipath_cur('scripts'))) + def_handlers.append(ss_part.ShellScriptPartHandler(self.paths.get_ipath_cur('scripts'))) if self.paths.get_ipath("boothooks"): - def_handlers.append(parts.BootHookPartHandler(self.paths.get_ipath("boothooks"))) + def_handlers.append(bh_part.BootHookPartHandler(self.paths.get_ipath("boothooks"))) if self.paths.upstart_conf_d: - def_handlers.append(parts.UpstartJobPartHandler(self.paths.upstart_conf_d)) + def_handlers.append(up_part.UpstartJobPartHandler(self.paths.upstart_conf_d)) return def_handlers def register_defaults(self): @@ -391,13 +396,11 @@ class CloudHandlers(object): class CloudConfig(object): def __init__(self, cfgfile, cloud): - self.cloud = cloud + self.cloud = CloudSimple(cloud) self.cfg = self._get_config(cfgfile) - self.paths = cloud.paths - self.sems = CloudSemaphores(self.paths) + self.sems = CloudSemaphores(self.cloud.paths) def _get_config(self, cfgfile): - cfg = None try: cfg = util.read_conf(cfgfile) @@ -430,5 +433,5 @@ class CloudConfig(object): if not freq: freq = def_freq c_name = "config-%s" % (name) - real_args = [name, self.cfg, self.cloud, LOG, args] + real_args = [name, copy.deepcopy(self.cfg), self.cloud, LOG, copy.deepcopy(args)] return self.sems.run_functor(c_name, freq, mod.handle, real_args) diff --git a/cloudinit/parts.py b/cloudinit/parts.py deleted file mode 100644 index 6af1ab7c..00000000 --- a/cloudinit/parts.py +++ /dev/null @@ -1,214 +0,0 @@ -import os - -from cloudinit import util -from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) -from cloudinit import log as logging - -LOG = logging.getLogger(__name__) - -CONTENT_END = "__end__" -CONTENT_START = "__begin__" -PART_CONTENT_TYPES = ["text/part-handler"] -PART_HANDLER_FN_TMPL = 'part-handler-%03d' - - -class PartHandler(object): - def __init__(self, frequency, version=2): - self.handler_version = version - self.frequency = frequency - - def __repr__(self): - return "%s: [%s]" % (self.__class__.__name__, self.list_types()) - - def list_types(self): - raise NotImplementedError() - - def handle_part(self, data, ctype, filename, payload, frequency): - return self._handle_part(data, ctype, filename, payload, frequency) - - def _handle_part(self, data, ctype, filename, payload, frequency): - raise NotImplementedError() - - -class BootHookPartHandler(PartHandler): - def __init__(self, boothook_dir, instance_id): - PartHandler.__init__(self, PER_ALWAYS) - self.boothook_dir = boothook_dir - self.instance_id = instance_id - - def list_types(self): - return ['text/cloud-boothook'] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype in [CONTENT_START, CONTENT_END]: - return - - filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - prefix = "#cloud-boothook" - start = 0 - if payload.startswith(prefix): - start = len(prefix) + 1 - - filepath = os.path.join(self.boothook_dir, filename) - util.write_file(filepath, payload[start:], 0700) - try: - env = os.environ.copy() - env['INSTANCE_ID'] = str(self.instance_id) - util.subp([filepath], env=env) - except util.ProcessExecutionError as e: - LOG.error("Boothooks script %s returned %s", filepath, e.exit_code) - except Exception as e: - LOG.error("Boothooks unknown exception %s when running %s", e, filepath) - - -class UpstartJobPartHandler(PartHandler): - def __init__(self, upstart_dir): - PartHandler.__init__(self, PER_INSTANCE) - self.upstart_dir = upstart_dir - - def list_types(self): - return ['text/upstart-job'] - - def _handle_part(self, _data, ctype, filename, payload, frequency): - if ctype in [CONTENT_START, CONTENT_END]: - return - - filename = utils.clean_filename(filename) - (name, ext) = os.path.splitext(filename) - ext = ext.lower() - if ext != ".conf": - filename = filename + ".conf" - - payload = util.dos2unix(payload) - util.write_file(os.path.join(self.upstart_dir, filename), payload, 0644) - - -class ShellScriptPartHandler(PartHandler): - - def __init__(self, script_dir): - PartHandler.__init__(self, PER_ALWAYS) - self.script_dir = script_dir - - def list_types(self): - return ['text/x-shellscript'] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype in [CONTENT_START, CONTENT_END]: - # maybe delete existing things here - return - - filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - util.write_file(os.path.join(self.script_dir, filename), payload, 0700) - - -class CloudConfigPartHandler(PartHandler): - def __init__(self, cloud_fn): - PartHandler.__init__(self, PER_ALWAYS) - self.cloud_buf = [] - self.cloud_fn = cloud_fn - - def list_types(self): - return ['text/cloud-config'] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype == CONTENT_START: - self.cloud_buf = [] - return - - if ctype == CONTENT_END: - payload = "\n".join(self.cloud_buf) - util.write_file(self.cloud_fn, payload, 0600) - self.cloud_buf = [] - return - - filename = util.clean_filename(filename) - entry = "\n".join(["#%s" % (filename), str(payload)]) - self.config_buf.append(entry) - - -def fixup_module(mod): - if not hasattr(mod, "handler_version"): - setattr(mod, "handler_version", 1) - if not hasattr(mod, 'list_types'): - def empty_types(): - return [] - setattr(mod, 'list_types', empty_types) - if not hasattr(mod, frequency): - setattr(mod, 'frequency', PER_INSTANCE) - if not hasattr(mod, 'handle_part'): - def empty_handler(data, ctype, filename, payload): - pass - setattr(mod, 'handle_part', empty_handler) - return mod - - -def find_module_files(root_dir): - entries = dict() - for fname in glob.glob(os.path.join(root_dir, "*.py")): - if not os.path.isfile(fname): - continue - modname = os.path.basename(fname)[0:-3] - entries[fname] = modname - return entries - - -def run_part(mod, data, ctype, filename, payload, frequency): - # only add the handler if the module should run - mod_freq = getattr(mod, "frequency") - if not (mod_freq == PER_ALWAYS or - (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): - return - try: - mod_ver = getattr(mod, 'handler_version') - if mod_ver == 1: - mod.handle_part(data, ctype, filename, payload) - else: - mod.handle_part(data, ctype, filename, payload, frequency) - except: - LOG.exception("Failed calling mod %s (%s, %s, %s) with frequency %s", mod, ctype, filename, mod_ver, frequency) - - -def call_begin(mod, data, frequency): - run_part(mod, data, CONTENT_START, None, None, frequency) - - -def call_end(mod, data, frequency): - run_part(mod, data, CONTENT_END, None, None, frequency) - - -def walker_handle_handler(pdata, _ctype, _filename, payload): - curcount = pdata['handlercount'] - modname = PART_HANDLER_FN_TMPL % (curcount) - frequency = pdata['frequency'] - modfname = os.path.join(pdata['handlerdir'], "%s.py" % (modname)) - # TODO: Check if path exists?? - util.write_file(modfname, payload, 0600) - handlers = pdata['handlers'] - try: - mod = fixup_module(importer.import_module(modname)) - handlers.register(mod) - call_begin(mod, pdata['data'], frequency) - pdata['handlercount'] = curcount + 1 - except: - LOG.exception("Failed at registered python file %s", modfname) - - -def walker_callback(pdata, ctype, filename, payload): - # data here is the part_handlers array and then the data to pass through - if ctype in PART_CONTENT_TYPES: - walker_handle_handler(pdata, ctype, filename, payload) - return - handlers = pdata['handlers'] - if ctype not in handlers: - if ctype == "text/x-not-multipart": - # Extract the first line or 24 bytes for displaying in the log - start = payload.split("\n", 1)[0][:24] - if start < payload: - details = "starting '%s...'" % start.encode("string-escape") - else: - details = repr(payload) - LOG.warning("Unhandled non-multipart userdata: %s", details) - return - run_part(handlers[ctype], pdata['data'], ctype, filename, payload, pdata['frequency']) diff --git a/cloudinit/parts/__init__.py b/cloudinit/parts/__init__.py new file mode 100644 index 00000000..20d4bd3b --- /dev/null +++ b/cloudinit/parts/__init__.py @@ -0,0 +1,117 @@ +import os + +from cloudinit import util +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) +from cloudinit import log as logging + +LOG = logging.getLogger(__name__) + +CONTENT_END = "__end__" +CONTENT_START = "__begin__" +PART_CONTENT_TYPES = ["text/part-handler"] +PART_HANDLER_FN_TMPL = 'part-handler-%03d' +UNDEF_HANDLER_VERSION = 1 + + +class PartHandler(object): + def __init__(self, frequency, version=2): + self.handler_version = version + self.frequency = frequency + + def __repr__(self): + return "%s: [%s]" % (self.__class__.__name__, self.list_types()) + + def list_types(self): + raise NotImplementedError() + + def handle_part(self, data, ctype, filename, payload, frequency): + return self._handle_part(data, ctype, filename, payload, frequency) + + def _handle_part(self, data, ctype, filename, payload, frequency): + raise NotImplementedError() + + +def fixup_module(mod): + if not hasattr(mod, "handler_version"): + setattr(mod, "handler_version", UNDEF_HANDLER_VERSION) + if not hasattr(mod, 'list_types'): + def empty_types(): + return [] + setattr(mod, 'list_types', empty_types) + if not hasattr(mod, frequency): + setattr(mod, 'frequency', PER_INSTANCE) + if not hasattr(mod, 'handle_part'): + def empty_handler(data, ctype, filename, payload): + pass + setattr(mod, 'handle_part', empty_handler) + return mod + + +def find_module_files(root_dir): + entries = dict() + for fname in glob.glob(os.path.join(root_dir, "*.py")): + if not os.path.isfile(fname): + continue + modname = os.path.basename(fname)[0:-3] + entries[fname] = modname + return entries + + +def run_part(mod, data, ctype, filename, payload, frequency): + # only add the handler if the module should run + mod_freq = getattr(mod, "frequency") + if not (mod_freq == PER_ALWAYS or + (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): + return + try: + mod_ver = getattr(mod, 'handler_version') + if mod_ver == 1: + mod.handle_part(data, ctype, filename, payload) + else: + mod.handle_part(data, ctype, filename, payload, frequency) + except: + LOG.exception("Failed calling mod %s (%s, %s, %s) with frequency %s", mod, ctype, filename, mod_ver, frequency) + + +def call_begin(mod, data, frequency): + run_part(mod, data, CONTENT_START, None, None, frequency) + + +def call_end(mod, data, frequency): + run_part(mod, data, CONTENT_END, None, None, frequency) + + +def walker_handle_handler(pdata, _ctype, _filename, payload): + curcount = pdata['handlercount'] + modname = PART_HANDLER_FN_TMPL % (curcount) + frequency = pdata['frequency'] + modfname = os.path.join(pdata['handlerdir'], "%s.py" % (modname)) + # TODO: Check if path exists?? + util.write_file(modfname, payload, 0600) + handlers = pdata['handlers'] + try: + mod = fixup_module(importer.import_module(modname)) + handlers.register(mod) + call_begin(mod, pdata['data'], frequency) + pdata['handlercount'] = curcount + 1 + except: + LOG.exception("Failed at registered python file %s", modfname) + + +def walker_callback(pdata, ctype, filename, payload): + # data here is the part_handlers array and then the data to pass through + if ctype in PART_CONTENT_TYPES: + walker_handle_handler(pdata, ctype, filename, payload) + return + handlers = pdata['handlers'] + if ctype not in handlers: + if ctype == "text/x-not-multipart": + # Extract the first line or 24 bytes for displaying in the log + start = payload.split("\n", 1)[0][:24] + if start < payload: + details = "starting '%s...'" % start.encode("string-escape") + else: + details = repr(payload) + LOG.warning("Unhandled non-multipart userdata: %s", details) + return + run_part(handlers[ctype], pdata['data'], ctype, filename, payload, pdata['frequency']) \ No newline at end of file diff --git a/cloudinit/parts/boot_hook.py b/cloudinit/parts/boot_hook.py new file mode 100644 index 00000000..881ffc58 --- /dev/null +++ b/cloudinit/parts/boot_hook.py @@ -0,0 +1,42 @@ +import os + +from cloudinit import util +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) +from cloudinit import log as logging +from cloudinit import parts + +LOG = logging.getLogger(__name__) + + + +class BootHookPartHandler(parts.PartHandler): + def __init__(self, boothook_dir, instance_id): + parts.PartHandler.__init__(self, PER_ALWAYS) + self.boothook_dir = boothook_dir + self.instance_id = instance_id + + def list_types(self): + return ['text/cloud-boothook'] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype in [CONTENT_START, CONTENT_END]: + return + + filename = util.clean_filename(filename) + payload = util.dos2unix(payload) + prefix = "#cloud-boothook" + start = 0 + if payload.startswith(prefix): + start = len(prefix) + 1 + + filepath = os.path.join(self.boothook_dir, filename) + util.write_file(filepath, payload[start:], 0700) + try: + env = os.environ.copy() + env['INSTANCE_ID'] = str(self.instance_id) + util.subp([filepath], env=env) + except util.ProcessExecutionError as e: + LOG.error("Boothooks script %s returned %s", filepath, e.exit_code) + except Exception as e: + LOG.error("Boothooks unknown exception %s when running %s", e, filepath) + diff --git a/cloudinit/parts/cloud_config.py b/cloudinit/parts/cloud_config.py new file mode 100644 index 00000000..dab0e5f5 --- /dev/null +++ b/cloudinit/parts/cloud_config.py @@ -0,0 +1,36 @@ +import os + +from cloudinit import util +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) +from cloudinit import log as logging +from cloudinit import parts + +LOG = logging.getLogger(__name__) + + + +class CloudConfigPartHandler(parts.PartHandler): + def __init__(self, cloud_fn): + parts.PartHandler.__init__(self, PER_ALWAYS) + self.cloud_buf = [] + self.cloud_fn = cloud_fn + + def list_types(self): + return ['text/cloud-config'] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype == CONTENT_START: + self.cloud_buf = [] + return + + if ctype == CONTENT_END: + payload = "\n".join(self.cloud_buf) + util.write_file(self.cloud_fn, payload, 0600) + self.cloud_buf = [] + return + + filename = util.clean_filename(filename) + entry = "\n".join(["#%s" % (filename), str(payload)]) + self.config_buf.append(entry) + + diff --git a/cloudinit/parts/shell_script.py b/cloudinit/parts/shell_script.py new file mode 100644 index 00000000..a248f198 --- /dev/null +++ b/cloudinit/parts/shell_script.py @@ -0,0 +1,27 @@ +import os + +from cloudinit import util +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) +from cloudinit import log as logging +from cloudinit import parts + +LOG = logging.getLogger(__name__) + + +class ShellScriptPartHandler(parts.PartHandler): + + def __init__(self, script_dir): + parts.PartHandler.__init__(self, PER_ALWAYS) + self.script_dir = script_dir + + def list_types(self): + return ['text/x-shellscript'] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype in [CONTENT_START, CONTENT_END]: + # maybe delete existing things here + return + + filename = util.clean_filename(filename) + payload = util.dos2unix(payload) + util.write_file(os.path.join(self.script_dir, filename), payload, 0700) diff --git a/cloudinit/parts/upstart_job.py b/cloudinit/parts/upstart_job.py new file mode 100644 index 00000000..7b290d26 --- /dev/null +++ b/cloudinit/parts/upstart_job.py @@ -0,0 +1,30 @@ +import os + +from cloudinit import util +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) +from cloudinit import log as logging +from cloudinit import parts + +LOG = logging.getLogger(__name__) + + +class UpstartJobPartHandler(parts.PartHandler): + def __init__(self, upstart_dir): + parts.PartHandler.__init__(self, PER_INSTANCE) + self.upstart_dir = upstart_dir + + def list_types(self): + return ['text/upstart-job'] + + def _handle_part(self, _data, ctype, filename, payload, frequency): + if ctype in [CONTENT_START, CONTENT_END]: + return + + filename = utils.clean_filename(filename) + (name, ext) = os.path.splitext(filename) + ext = ext.lower() + if ext != ".conf": + filename = filename + ".conf" + + payload = util.dos2unix(payload) + util.write_file(os.path.join(self.upstart_dir, filename), payload, 0644) -- cgit v1.2.3