From 9730b1470f029a514cacbba197c7946d8fedf3d4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 20 Feb 2014 13:11:38 -0500 Subject: initial commit for status --- bin/cloud-init | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 2 deletions(-) (limited to 'bin/cloud-init') diff --git a/bin/cloud-init b/bin/cloud-init index 80a1df05..e22f54de 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -418,6 +418,92 @@ def main_single(name, args): # Guess it worked return 0 +def status_wrapper(args): + (name, functor) = args.action + + if args.name: + if args.local: + mode = "init-local" + else: + mode = "init" + elif args.name == "modules": + mode = "modules-%s" % args.mode + + modes = ('init', 'init-local', 'modules-config', 'modules-final') + + if mode == 'init': + nullstatus = { + 'errors': [] + 'state': None + 'start': None + 'end': None + } + status = {'v1': {}} + for mode in modes: + status['v1'][mode] = nullstatus.copy() + else: + status = load_status() + status['stage'] = mode + + v1 = status['v1'] + v1[mode]['start'] = time.time() + update_status(status) + # status + # { + # 'v1': { + # 'init': { + # errors: [] + # start: + # end: + # }, + # 'init-local': { + # errors: [] + # start: + # end: + # }, + # 'modules-final': { + # }, + # 'modules-config': { + # }, + # 'datasource': None + # 'stage': ('init', 'init-local', 'modules-final', 'modules-config', 'finished') + # 'errors': + # } + # finished + # { + # 'datasource': + # 'errors': + # } + # + # + exception = None + try: + ret = func(args) + except Exception as e: + v1[mode]['errors'] = [str(e)] + + v1[mode]['finished'] = time.time() + v1['stage'] = None + + + if mode in ('init' or 'init-local'): + # FIXME(smoser): add the datasource here + v1['datasource'] = "~~~datasource~~~" + + update_status(status) + + if mode == "modules-final": + # write the 'finished' file + errors = [] + for m in modes: + if v1[m]['errors']: + errors += v1[m]['errors'] + + finished = {'datasource': v1['datasource'], + 'errors': errors} + + return ret + def main(): parser = argparse.ArgumentParser() @@ -450,7 +536,7 @@ def main(): default=False) # This is used so that we can know which action is selected + # the functor to use to run this subcommand - parser_init.set_defaults(action=('init', main_init)) + parser_init.set_defaults(action=('init', status_wrapper)) # These settings are used for the 'config' and 'final' stages parser_mod = subparsers.add_parser('modules', @@ -461,7 +547,7 @@ def main(): "to use (default: %(default)s)"), default='config', choices=('init', 'config', 'final')) - parser_mod.set_defaults(action=('modules', main_modules)) + parser_mod.set_defaults(action=('modules', status_wrapper)) # These settings are used when you want to query information # stored in the cloud-init data objects/directories/files -- cgit v1.2.3 From 2781fc8289e4aab130125b1a3e69b45a9318f805 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 24 Feb 2014 16:27:28 -0500 Subject: possibly functional start testing --- bin/cloud-init | 128 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 66 insertions(+), 62 deletions(-) (limited to 'bin/cloud-init') diff --git a/bin/cloud-init b/bin/cloud-init index e22f54de..dc480901 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -22,8 +22,10 @@ # along with this program. If not, see . import argparse +import json import os import sys +import time import traceback # This is more just for running from the bin folder so that @@ -126,11 +128,11 @@ def run_module_section(mods, action_name, section): " under section '%s'") % (action_name, full_section_name) sys.stderr.write("%s\n" % (msg)) LOG.debug(msg) - return 0 + return [] else: LOG.debug("Ran %s modules with %s failures", len(which_ran), len(failures)) - return len(failures) + return failures def main_init(name, args): @@ -220,7 +222,7 @@ def main_init(name, args): if existing_files: LOG.debug("Exiting early due to the existence of %s files", existing_files) - return 0 + return (None, []) else: # The cache is not instance specific, so it has to be purged # but we want 'start' to benefit from a cache if @@ -249,9 +251,9 @@ def main_init(name, args): " Likely bad things to come!")) if not args.force: if args.local: - return 0 + return (None, []) else: - return 1 + return (None, ["No instance datasource found."]) # Stage 6 iid = init.instancify() LOG.debug("%s will now be targeting instance id: %s", name, iid) @@ -274,7 +276,7 @@ def main_init(name, args): init.consume_data(PER_ALWAYS) except Exception: util.logexc(LOG, "Consuming user data failed!") - return 1 + return (init.datasource, ["Consuming user data failed!"]) # Stage 8 - re-read and apply relevant cloud-config to include user-data mods = stages.Modules(init, extract_fns(args)) @@ -291,7 +293,7 @@ def main_init(name, args): logging.setupLogging(mods.cfg) # Stage 10 - return run_module_section(mods, name, name) + return (init.datasource, run_module_section(mods, name, name)) def main_modules(action_name, args): @@ -315,14 +317,12 @@ def main_modules(action_name, args): init.fetch() except sources.DataSourceNotFoundException: # There was no datasource found, theres nothing to do - util.logexc(LOG, ('Can not apply stage %s, ' - 'no datasource found!' - " Likely bad things to come!"), name) - print_exc(('Can not apply stage %s, ' - 'no datasource found!' - " Likely bad things to come!") % (name)) + msg = ('Can not apply stage %s, no datasource found! Likely bad ' + 'things to come!' % name) + util.logexc(LOG, msg) + print_exc(msg) if not args.force: - return 1 + return [(msg)] # Stage 3 mods = stages.Modules(init, extract_fns(args)) # Stage 4 @@ -418,8 +418,21 @@ def main_single(name, args): # Guess it worked return 0 -def status_wrapper(args): - (name, functor) = args.action + +def status_wrapper(args, data_d=None, link_d=None): + if data_d is None: + data_d = os.path.normpath("/var/lib/cloud/data") + if link_d is None: + link_d = os.path.normpath("/run/cloud-init") + + status_path = os.path.join(data_d, "status.json") + status_link = os.path.join(link_d, "status.json") + result_path = os.path.join(data_d, "result.json") + result_link = os.path.join(link_d, "result.json") + + util.ensure_dirs((data_d, link_d,)) + + (_name, functor) = args.action if args.name: if args.local: @@ -431,78 +444,69 @@ def status_wrapper(args): modes = ('init', 'init-local', 'modules-config', 'modules-final') - if mode == 'init': + status = None + if mode == 'init-local': + for f in (status_link, result_link, status_path, result_path): + util.del_file(f) + else: + try: + status = json.loads(util.load_file(status_path)) + except: + pass + + if status is None: nullstatus = { - 'errors': [] - 'state': None - 'start': None - 'end': None + 'errors': [], + 'state': None, + 'start': None, + 'end': None, } status = {'v1': {}} for mode in modes: status['v1'][mode] = nullstatus.copy() - else: - status = load_status() + status['v1']['datasource'] = None + status['stage'] = mode v1 = status['v1'] v1[mode]['start'] = time.time() - update_status(status) - # status - # { - # 'v1': { - # 'init': { - # errors: [] - # start: - # end: - # }, - # 'init-local': { - # errors: [] - # start: - # end: - # }, - # 'modules-final': { - # }, - # 'modules-config': { - # }, - # 'datasource': None - # 'stage': ('init', 'init-local', 'modules-final', 'modules-config', 'finished') - # 'errors': - # } - # finished - # { - # 'datasource': - # 'errors': - # } - # - # - exception = None + + util.write_file(status_path, json.dumps(status)) + util.sym_link(os.path.relpath(os.path.status_path, link_d), status_link) + try: - ret = func(args) + ret = functor(args) except Exception as e: v1[mode]['errors'] = [str(e)] v1[mode]['finished'] = time.time() v1['stage'] = None + if mode in ('init', 'init-local'): + (datasource, errors) = ret + if datasource is not None: + v1['datasource'] = datasource + v1[mode]['errors'] = errors + else: + errors = ret + v1[mode]['errors'] = ret - if mode in ('init' or 'init-local'): - # FIXME(smoser): add the datasource here - v1['datasource'] = "~~~datasource~~~" - - update_status(status) + util.write_file(status_path, json.dumps(status)) if mode == "modules-final": # write the 'finished' file errors = [] for m in modes: if v1[m]['errors']: - errors += v1[m]['errors'] - + errors.extend(v1[m].get('errors', [])) + finished = {'datasource': v1['datasource'], 'errors': errors} + util.write_file(result_path, json.dumps(finished)) + util.sym_link(os.path.relpath(os.path.result_path, link_d), + result_link) - return ret + return len(v1[mode]['errors']) def main(): -- cgit v1.2.3 From da13f065c9a2be372fea35db62e51086d443f8dc Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 24 Feb 2014 17:20:12 -0500 Subject: fixes from testing, force symlink --- bin/cloud-init | 48 ++++++++++++++++++++++++++---------------------- cloudinit/util.py | 4 +++- 2 files changed, 29 insertions(+), 23 deletions(-) (limited to 'bin/cloud-init') diff --git a/bin/cloud-init b/bin/cloud-init index dc480901..78f8600d 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -419,7 +419,7 @@ def main_single(name, args): return 0 -def status_wrapper(args, data_d=None, link_d=None): +def status_wrapper(name, args, data_d=None, link_d=None): if data_d is None: data_d = os.path.normpath("/var/lib/cloud/data") if link_d is None: @@ -434,13 +434,15 @@ def status_wrapper(args, data_d=None, link_d=None): (_name, functor) = args.action - if args.name: + if name == "init": if args.local: mode = "init-local" else: mode = "init" - elif args.name == "modules": + elif name == "modules": mode = "modules-%s" % args.mode + else: + raise ValueError("unknown name: %s" % name) modes = ('init', 'init-local', 'modules-config', 'modules-final') @@ -457,40 +459,40 @@ def status_wrapper(args, data_d=None, link_d=None): if status is None: nullstatus = { 'errors': [], - 'state': None, 'start': None, 'end': None, } status = {'v1': {}} - for mode in modes: - status['v1'][mode] = nullstatus.copy() + for m in modes: + status['v1'][m] = nullstatus.copy() status['v1']['datasource'] = None - status['stage'] = mode v1 = status['v1'] + v1['stage'] = mode v1[mode]['start'] = time.time() util.write_file(status_path, json.dumps(status)) - util.sym_link(os.path.relpath(os.path.status_path, link_d), status_link) + util.sym_link(os.path.relpath(status_path, link_d), status_link, + force=True) try: - ret = functor(args) + ret = functor(name, args) + if mode in ('init', 'init-local'): + (datasource, errors) = ret + if datasource is not None: + v1['datasource'] = datasource + v1[mode]['errors'] = errors + else: + errors = ret + v1[mode]['errors'] = ret + except Exception as e: v1[mode]['errors'] = [str(e)] v1[mode]['finished'] = time.time() v1['stage'] = None - if mode in ('init', 'init-local'): - (datasource, errors) = ret - if datasource is not None: - v1['datasource'] = datasource - v1[mode]['errors'] = errors - else: - errors = ret - v1[mode]['errors'] = ret - util.write_file(status_path, json.dumps(status)) if mode == "modules-final": @@ -503,8 +505,8 @@ def status_wrapper(args, data_d=None, link_d=None): finished = {'datasource': v1['datasource'], 'errors': errors} util.write_file(result_path, json.dumps(finished)) - util.sym_link(os.path.relpath(os.path.result_path, link_d), - result_link) + util.sym_link(os.path.relpath(result_path, link_d), result_link, + force=True) return len(v1[mode]['errors']) @@ -540,7 +542,7 @@ def main(): default=False) # This is used so that we can know which action is selected + # the functor to use to run this subcommand - parser_init.set_defaults(action=('init', status_wrapper)) + parser_init.set_defaults(action=('init', main_init)) # These settings are used for the 'config' and 'final' stages parser_mod = subparsers.add_parser('modules', @@ -551,7 +553,7 @@ def main(): "to use (default: %(default)s)"), default='config', choices=('init', 'config', 'final')) - parser_mod.set_defaults(action=('modules', status_wrapper)) + parser_mod.set_defaults(action=('modules', main_modules)) # These settings are used when you want to query information # stored in the cloud-init data objects/directories/files @@ -592,6 +594,8 @@ def main(): signal_handler.attach_handlers() (name, functor) = args.action + if name in ("modules", "init"): + functor = status_wrapper return util.log_time(logfunc=LOG.debug, msg="cloud-init mode '%s'" % name, get_uptime=True, func=functor, args=(name, args)) diff --git a/cloudinit/util.py b/cloudinit/util.py index 87b0c853..06039ee2 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1395,8 +1395,10 @@ def get_builtin_cfg(): return obj_copy.deepcopy(CFG_BUILTIN) -def sym_link(source, link): +def sym_link(source, link, force=False): LOG.debug("Creating symbolic link from %r => %r", link, source) + if force and os.path.exists(link): + del_file(link) os.symlink(source, link) -- cgit v1.2.3 From af6b25e0b8895e8eead0a7202d637fa197c4401c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 24 Feb 2014 20:47:35 -0500 Subject: minor cleanups --- bin/cloud-init | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'bin/cloud-init') diff --git a/bin/cloud-init b/bin/cloud-init index 78f8600d..479d715d 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -419,6 +419,10 @@ def main_single(name, args): return 0 +def write_json(path, data): + util.write_file(path, json.dumps(data, indent=1) + "\n") + + def status_wrapper(name, args, data_d=None, link_d=None): if data_d is None: data_d = os.path.normpath("/var/lib/cloud/data") @@ -472,7 +476,7 @@ def status_wrapper(name, args, data_d=None, link_d=None): v1['stage'] = mode v1[mode]['start'] = time.time() - util.write_file(status_path, json.dumps(status)) + write_json(status, status_path) util.sym_link(os.path.relpath(status_path, link_d), status_link, force=True) @@ -481,11 +485,11 @@ def status_wrapper(name, args, data_d=None, link_d=None): if mode in ('init', 'init-local'): (datasource, errors) = ret if datasource is not None: - v1['datasource'] = datasource - v1[mode]['errors'] = errors + v1['datasource'] = str(datasource) else: errors = ret - v1[mode]['errors'] = ret + + v1[mode]['errors'] = [str(e) for e in errors] except Exception as e: v1[mode]['errors'] = [str(e)] @@ -493,7 +497,7 @@ def status_wrapper(name, args, data_d=None, link_d=None): v1[mode]['finished'] = time.time() v1['stage'] = None - util.write_file(status_path, json.dumps(status)) + write_json(status_path, status) if mode == "modules-final": # write the 'finished' file @@ -502,9 +506,8 @@ def status_wrapper(name, args, data_d=None, link_d=None): if v1[m]['errors']: errors.extend(v1[m].get('errors', [])) - finished = {'datasource': v1['datasource'], - 'errors': errors} - util.write_file(result_path, json.dumps(finished)) + write_json(result_path, + {'datasource': v1['datasource'], 'errors': errors}) util.sym_link(os.path.relpath(result_path, link_d), result_link, force=True) -- cgit v1.2.3 From 1692175c0b0afe543065303251674f77e925e2a9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 24 Feb 2014 20:49:22 -0500 Subject: fix write_json call --- bin/cloud-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/cloud-init') diff --git a/bin/cloud-init b/bin/cloud-init index 479d715d..d1cd68ea 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -476,7 +476,7 @@ def status_wrapper(name, args, data_d=None, link_d=None): v1['stage'] = mode v1[mode]['start'] = time.time() - write_json(status, status_path) + write_json(status_path, status) util.sym_link(os.path.relpath(status_path, link_d), status_link, force=True) -- cgit v1.2.3 From 27081dacc0812be242860e31f0473b69e7c45c49 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 25 Feb 2014 12:07:03 -0500 Subject: be atomic when writing status files --- bin/cloud-init | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) (limited to 'bin/cloud-init') diff --git a/bin/cloud-init b/bin/cloud-init index d1cd68ea..261aaa4e 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -26,6 +26,7 @@ import json import os import sys import time +import tempfile import traceback # This is more just for running from the bin folder so that @@ -419,8 +420,18 @@ def main_single(name, args): return 0 -def write_json(path, data): - util.write_file(path, json.dumps(data, indent=1) + "\n") +def atomic_write_json(path, data): + tf = None + try: + tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path), + delete=False) + tf.write(json.dumps(data, indent=1) + "\n") + tf.close() + os.rename(tf.name, path) + except Exception as e: + if tf is not None: + util.del_file(tf.name) + raise e def status_wrapper(name, args, data_d=None, link_d=None): @@ -471,12 +482,11 @@ def status_wrapper(name, args, data_d=None, link_d=None): status['v1'][m] = nullstatus.copy() status['v1']['datasource'] = None - v1 = status['v1'] v1['stage'] = mode v1[mode]['start'] = time.time() - write_json(status_path, status) + atomic_write_json(status_path, status) util.sym_link(os.path.relpath(status_path, link_d), status_link, force=True) @@ -497,7 +507,7 @@ def status_wrapper(name, args, data_d=None, link_d=None): v1[mode]['finished'] = time.time() v1['stage'] = None - write_json(status_path, status) + atomic_write_json(status_path, status) if mode == "modules-final": # write the 'finished' file @@ -506,7 +516,7 @@ def status_wrapper(name, args, data_d=None, link_d=None): if v1[m]['errors']: errors.extend(v1[m].get('errors', [])) - write_json(result_path, + atomic_write_json(result_path, {'datasource': v1['datasource'], 'errors': errors}) util.sym_link(os.path.relpath(result_path, link_d), result_link, force=True) -- cgit v1.2.3 From f33583bae55dbf071cce88c4e85b289c93e970c8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 3 Mar 2014 16:49:37 -0500 Subject: version space (v1:) result_path json also --- bin/cloud-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin/cloud-init') diff --git a/bin/cloud-init b/bin/cloud-init index 261aaa4e..6ede60af 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -517,7 +517,7 @@ def status_wrapper(name, args, data_d=None, link_d=None): errors.extend(v1[m].get('errors', [])) atomic_write_json(result_path, - {'datasource': v1['datasource'], 'errors': errors}) + {'v1': {'datasource': v1['datasource'], 'errors': errors}}) util.sym_link(os.path.relpath(result_path, link_d), result_link, force=True) -- cgit v1.2.3