From f895cb12141281702b34da18f2384deb64c881e7 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 21 Jan 2015 17:56:53 -0500 Subject: Largely merge lp:~harlowja/cloud-init/py2-3 albeit manually because it seemed to be behind trunk. `tox -e py27` passes full test suite. Now to work on replacing mocker. --- cloudinit/stages.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 67f467f7..f4f4591d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -20,12 +20,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cPickle as pickle - import copy import os import sys +import six +from six.moves import cPickle as pickle + from cloudinit.settings import (PER_INSTANCE, FREQUENCIES, CLOUD_CONFIG) from cloudinit import handlers @@ -202,7 +203,7 @@ class Init(object): util.logexc(LOG, "Failed pickling datasource %s", self.datasource) return False try: - util.write_file(pickled_fn, pk_contents, mode=0400) + util.write_file(pickled_fn, pk_contents, mode=0o400) except Exception: util.logexc(LOG, "Failed pickling datasource to %s", pickled_fn) return False @@ -324,15 +325,15 @@ class Init(object): def _store_userdata(self): raw_ud = "%s" % (self.datasource.get_userdata_raw()) - util.write_file(self._get_ipath('userdata_raw'), raw_ud, 0600) + util.write_file(self._get_ipath('userdata_raw'), raw_ud, 0o600) processed_ud = "%s" % (self.datasource.get_userdata()) - util.write_file(self._get_ipath('userdata'), processed_ud, 0600) + util.write_file(self._get_ipath('userdata'), processed_ud, 0o600) def _store_vendordata(self): raw_vd = "%s" % (self.datasource.get_vendordata_raw()) - util.write_file(self._get_ipath('vendordata_raw'), raw_vd, 0600) + util.write_file(self._get_ipath('vendordata_raw'), raw_vd, 0o600) processed_vd = "%s" % (self.datasource.get_vendordata()) - util.write_file(self._get_ipath('vendordata'), processed_vd, 0600) + util.write_file(self._get_ipath('vendordata'), processed_vd, 0o600) def _default_handlers(self, opts=None): if opts is None: @@ -384,7 +385,7 @@ class Init(object): if not path or not os.path.isdir(path): return potential_handlers = util.find_modules(path) - for (fname, mod_name) in potential_handlers.iteritems(): + for (fname, mod_name) in potential_handlers.items(): try: mod_locs, looked_locs = importer.find_module( mod_name, [''], ['list_types', 'handle_part']) @@ -422,7 +423,7 @@ class Init(object): def init_handlers(): # Init the handlers first - for (_ctype, mod) in c_handlers.iteritems(): + for (_ctype, mod) in c_handlers.items(): if mod in c_handlers.initialized: # Avoid initing the same module twice (if said module # is registered to more than one content-type). @@ -449,7 +450,7 @@ class Init(object): def finalize_handlers(): # Give callbacks opportunity to finalize - for (_ctype, mod) in c_handlers.iteritems(): + for (_ctype, mod) in c_handlers.items(): if mod not in c_handlers.initialized: # Said module was never inited in the first place, so lets # not attempt to finalize those that never got called. @@ -574,7 +575,7 @@ class Modules(object): for item in cfg_mods: if not item: continue - if isinstance(item, (str, basestring)): + if isinstance(item, six.string_types): module_list.append({ 'mod': item.strip(), }) -- cgit v1.2.3 From 9acda162465a9b580a79a0fdc2c8e003151c7ead Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 11 Feb 2015 13:45:55 -0500 Subject: pickle contents: be careful loading and storing pickle to be binary --- cloudinit/stages.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index f4f4591d..c5b1ded0 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -180,9 +180,12 @@ class Init(object): pickled_fn = self.paths.get_ipath_cur('obj_pkl') pickle_contents = None try: - pickle_contents = util.load_file(pickled_fn) - except Exception: + pickle_contents = util.load_file(pickled_fn, decode=False) + except Exception as e: + if os.path.isfile(pickled_fn): + LOG.warn("failed loading pickle in %s: %s" % (pickled_fn, e)) pass + # This is expected so just return nothing # successfully loaded... if not pickle_contents: @@ -203,7 +206,7 @@ class Init(object): util.logexc(LOG, "Failed pickling datasource %s", self.datasource) return False try: - util.write_file(pickled_fn, pk_contents, mode=0o400) + util.write_file(pickled_fn, pk_contents, omode="wb", mode=0o400) except Exception: util.logexc(LOG, "Failed pickling datasource to %s", pickled_fn) return False -- cgit v1.2.3 From e2fea567772f3d178072607aee617c3792185db0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 24 Feb 2015 16:19:02 -0500 Subject: further fixing of non-text user-data. --- cloudinit/stages.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index c5b1ded0..94fcf4cc 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -327,16 +327,26 @@ class Init(object): self._store_vendordata() def _store_userdata(self): - raw_ud = "%s" % (self.datasource.get_userdata_raw()) + raw_ud = self.datasource.get_userdata_raw() + if raw_ud is None: + raw_ud = b'' util.write_file(self._get_ipath('userdata_raw'), raw_ud, 0o600) - processed_ud = "%s" % (self.datasource.get_userdata()) - util.write_file(self._get_ipath('userdata'), processed_ud, 0o600) + # processed userdata is a Mime message, so write it as string. + processed_ud = self.datasource.get_userdata() + if processed_ud is None: + raw_ud = '' + util.write_file(self._get_ipath('userdata'), str(processed_ud), 0o600) def _store_vendordata(self): - raw_vd = "%s" % (self.datasource.get_vendordata_raw()) + raw_vd = self.datasource.get_vendordata_raw() + if raw_vd is None: + raw_vd = b'' util.write_file(self._get_ipath('vendordata_raw'), raw_vd, 0o600) - processed_vd = "%s" % (self.datasource.get_vendordata()) - util.write_file(self._get_ipath('vendordata'), processed_vd, 0o600) + # processed vendor data is a Mime message, so write it as string. + processed_vd = str(self.datasource.get_vendordata()) + if processed_vd is None: + processed_vd = '' + util.write_file(self._get_ipath('vendordata'), str(processed_vd), 0o600) def _default_handlers(self, opts=None): if opts is None: -- cgit v1.2.3 From f0388cfffadc0596faeda9e11775597444bff25d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 2 Mar 2015 15:41:16 -0500 Subject: pep8 --- cloudinit/ec2_utils.py | 1 + cloudinit/stages.py | 3 ++- tests/unittests/test_data.py | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 7cf99186..37b92a83 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -185,6 +185,7 @@ def get_instance_metadata(api_version='latest', caller = functools.partial(util.read_file_or_url, ssl_details=ssl_details, timeout=timeout, retries=retries) + def mcaller(url): return caller(url).contents diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 94fcf4cc..45d64823 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -346,7 +346,8 @@ class Init(object): processed_vd = str(self.datasource.get_vendordata()) if processed_vd is None: processed_vd = '' - util.write_file(self._get_ipath('vendordata'), str(processed_vd), 0o600) + util.write_file(self._get_ipath('vendordata'), str(processed_vd), + 0o600) def _default_handlers(self, opts=None): if opts is None: diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 48475515..8fc280e4 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -58,7 +58,6 @@ def gzip_text(text): return contents.getvalue() - # FIXME: these tests shouldn't be checking log output?? # Weirddddd... class TestConsumeUserData(helpers.FilesystemMockingTestCase): -- cgit v1.2.3 From ac445690ab7a69ac6e9c74b136de9968b0c839df Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 4 Mar 2015 12:42:34 -0500 Subject: fix logging perms with list rather than single --- cloudinit/settings.py | 2 +- cloudinit/stages.py | 21 +++++++++++++++------ doc/examples/cloud-config.txt | 2 ++ 3 files changed, 18 insertions(+), 7 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 5efcb0b0..b61e5613 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -47,7 +47,7 @@ CFG_BUILTIN = { ], 'def_log_file': '/var/log/cloud-init.log', 'log_cfgs': [], - 'syslog_fix_perms': 'syslog:adm', + 'syslog_fix_perms': ['syslog:adm', 'root:adm'], 'system_info': { 'paths': { 'cloud_dir': '/var/lib/cloud', diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 45d64823..d28e765b 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -148,16 +148,25 @@ class Init(object): def _initialize_filesystem(self): util.ensure_dirs(self._initial_subdirs()) log_file = util.get_cfg_option_str(self.cfg, 'def_log_file') - perms = util.get_cfg_option_str(self.cfg, 'syslog_fix_perms') if log_file: util.ensure_file(log_file) - if perms: - u, g = util.extract_usergroup(perms) + perms = self.cfg.get('syslog_fix_perms') + if not perms: + perms = {} + if not isinstance(perms, list): + perms = [perms] + + error = None + for perm in perms: + u, g = util.extract_usergroup(perm) try: util.chownbyname(log_file, u, g) - except OSError: - util.logexc(LOG, "Unable to change the ownership of %s to " - "user %s, group %s", log_file, u, g) + return + except OSError as e: + error = e + + LOG.warn("Failed changing perms on '%s'. tried: %s. %s", + log_file, ','.join(perms), error) def read_cfg(self, extra_fns=None): # None check so that we don't keep on re-loading if empty diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 1c59c2cf..1236796c 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -536,6 +536,8 @@ timezone: US/Eastern # # to remedy this situation, 'def_log_file' can be set to a filename # and syslog_fix_perms to a string containing ":" +# if syslog_fix_perms is a list, it will iterate through and use the +# first pair that does not raise error. # # the default values are '/var/log/cloud-init.log' and 'syslog:adm' # the value of 'def_log_file' should match what is configured in logging -- cgit v1.2.3 From b5574a9925b29417a1b351e7b38c54bc7d144dba Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 30 Jul 2015 18:06:01 -0400 Subject: tests pass --- bin/cloud-init | 28 ++++++++++-- cloudinit/reporting.py | 91 +++++++++++++++++++++++++++++++++++---- cloudinit/sources/__init__.py | 16 ++++--- cloudinit/stages.py | 10 ++++- tests/unittests/test_reporting.py | 14 +++--- 5 files changed, 134 insertions(+), 25 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/bin/cloud-init b/bin/cloud-init index 1d3e7ee3..7f21e49f 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -46,6 +46,7 @@ from cloudinit import sources from cloudinit import stages from cloudinit import templater from cloudinit import util +from cloudinit import reporting from cloudinit import version from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, @@ -313,7 +314,7 @@ def main_modules(action_name, args): # 5. Run the modules for the given stage name # 6. Done! w_msg = welcome_format("%s:%s" % (action_name, name)) - init = stages.Init(ds_deps=[]) + init = stages.Init(ds_deps=[], reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 @@ -549,6 +550,8 @@ def main(): ' found (use at your own risk)'), dest='force', default=False) + + parser.set_defaults(reporter=None) subparsers = parser.add_subparsers() # Each action and its sub-options (if any) @@ -595,6 +598,9 @@ def main(): help=("frequency of the module"), required=False, choices=list(FREQ_SHORT_NAMES.keys())) + parser_single.add_argument("--report", action="store_true", + help="enable reporting", + required=False) parser_single.add_argument("module_args", nargs="*", metavar='argument', help=('any additional arguments to' @@ -617,8 +623,24 @@ def main(): if name in ("modules", "init"): functor = status_wrapper - return util.log_time(logfunc=LOG.debug, msg="cloud-init mode '%s'" % name, - get_uptime=True, func=functor, args=(name, args)) + reporting = True + if name == "init": + if args.local: + rname, rdesc = ("init-local", "searching for local datasources") + else: + rname, rdesc = ("init-network", "searching for network datasources") + elif name == "modules": + rname, rdesc = ("modules-%s" % args.mode, "running modules for %s") + elif name == "single": + rname, rdesc = ("single/%s" % args.name, + "running single module %s" % args.name) + reporting = args.report + + reporter = reporting.ReportStack(rname, rdesc, reporting=reporting) + with reporter: + return util.log_time( + logfunc=LOG.debug, msg="cloud-init mode '%s'" % name, + get_uptime=True, func=functor, args=(name, args)) if __name__ == '__main__': diff --git a/cloudinit/reporting.py b/cloudinit/reporting.py index d2dd4fec..c925f661 100644 --- a/cloudinit/reporting.py +++ b/cloudinit/reporting.py @@ -20,9 +20,18 @@ START_EVENT_TYPE = 'start' DEFAULT_CONFIG = { 'logging': {'type': 'log'}, + 'print': {'type': 'print'}, } +class _nameset(set): + def __getattr__(self, name): + if name in self: + return name + raise AttributeError + +status = _nameset(("SUCCESS", "WARN", "FAIL")) + instantiated_handler_registry = DictRegistry() available_handlers = DictRegistry() @@ -43,17 +52,18 @@ class ReportingEvent(object): class FinishReportingEvent(ReportingEvent): - def __init__(self, name, description, successful=None): + def __init__(self, name, description, result=None): super(FinishReportingEvent, self).__init__( FINISH_EVENT_TYPE, name, description) - self.successful = successful + if result is None: + result = status.SUCCESS + self.result = result + if result not in status: + raise ValueError("Invalid result: %s" % result) def as_string(self): - if self.successful is None: - return super(FinishReportingEvent, self).as_string() - success_string = 'success' if self.successful else 'fail' return '{0}: {1}: {2}: {3}'.format( - self.event_type, self.name, success_string, self.description) + self.event_type, self.name, self.result, self.description) class ReportingHandler(object): @@ -73,6 +83,11 @@ class LogHandler(ReportingHandler): logger.info(event.as_string()) +class PrintHandler(ReportingHandler): + def publish_event(self, event): + print(event.as_string()) + + def add_configuration(config): for handler_name, handler_config in config.items(): handler_config = handler_config.copy() @@ -95,12 +110,12 @@ def report_event(event): handler.publish_event(event) -def report_finish_event(event_name, event_description, successful=None): +def report_finish_event(event_name, event_description, result): """Report a "finish" event. See :py:func:`.report_event` for parameter details. """ - event = FinishReportingEvent(event_name, event_description, successful) + event = FinishReportingEvent(event_name, event_description, result) return report_event(event) @@ -118,5 +133,65 @@ def report_start_event(event_name, event_description): return report_event(event) +class ReportStack(object): + def __init__(self, name, description, parent=None, reporting=None, + exc_result=None): + self.parent = parent + self.reporting = reporting + self.name = name + self.description = description + + if exc_result is None: + exc_result = status.FAIL + self.exc_result = exc_result + + if reporting is None: + # if reporting is specified respect it, otherwise use parent's value + if parent: + reporting = parent.reporting + else: + reporting = True + if parent: + self.fullname = '/'.join((name, parent.fullname,)) + else: + self.fullname = self.name + self.children = {} + + def __enter__(self): + self.exception = None + if self.reporting: + report_start_event(self.fullname, self.description) + if self.parent: + self.parent.children[self.name] = (None, None) + return self + + def childrens_finish_info(self, result=None, description=None): + for result in (status.FAIL, status.WARN): + for name, (value, msg) in self.children.items(): + if value == result: + return (result, "[" + name + "]" + msg) + if result is None: + result = status.SUCCESS + if description is None: + description = self.description + return (result, description) + + def finish_info(self, exc): + # return tuple of description, and value + if exc: + # by default, exceptions are fatal + return (self.exc_result, self.description) + return self.childrens_finish_info() + + def __exit__(self, exc_type, exc_value, traceback): + self.exception = exc_value + (result, msg) = self.finish_info(exc_value) + if self.parent: + self.parent.children[self.name] = (result, msg) + if self.reporting: + report_finish_event(self.fullname, msg, result) + + available_handlers.register_item('log', LogHandler) +available_handlers.register_item('print', PrintHandler) add_configuration(DEFAULT_CONFIG) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index a21c08c2..c4848d5d 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -27,6 +27,7 @@ import six from cloudinit import importer from cloudinit import log as logging +from cloudinit import reporting from cloudinit import type_utils from cloudinit import user_data as ud from cloudinit import util @@ -246,17 +247,22 @@ def normalize_pubkey_data(pubkey_data): return keys -def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): +def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter): ds_list = list_sources(cfg_list, ds_deps, pkg_list) ds_names = [type_utils.obj_name(f) for f in ds_list] LOG.debug("Searching for data source in: %s", ds_names) for cls in ds_list: + myreporter = reporting.ReportStack( + "check-%s" % cls, "searching for %s" % cls, + parent=reporter, exc_result=reporting.status.WARN) + try: - LOG.debug("Seeing if we can get any data from %s", cls) - s = cls(sys_cfg, distro, paths) - if s.get_data(): - return (s, type_utils.obj_name(cls)) + with myreporter: + LOG.debug("Seeing if we can get any data from %s", cls) + s = cls(sys_cfg, distro, paths) + if s.get_data(): + return (s, type_utils.obj_name(cls)) except Exception: util.logexc(LOG, "Getting data from %s failed", cls) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index d28e765b..dbcdbece 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -46,6 +46,7 @@ from cloudinit import log as logging from cloudinit import sources from cloudinit import type_utils from cloudinit import util +from cloudinit import reporting LOG = logging.getLogger(__name__) @@ -53,7 +54,7 @@ NULL_DATA_SOURCE = None class Init(object): - def __init__(self, ds_deps=None): + def __init__(self, reporter=None, ds_deps=None): if ds_deps is not None: self.ds_deps = ds_deps else: @@ -65,6 +66,11 @@ class Init(object): # Changed only when a fetch occurs self.datasource = NULL_DATA_SOURCE + if reporter is None: + reporter = reporting.ReportStack( + name="init-reporter", description="init-desc", reporting=False) + self.reporter = reporter + def _reset(self, reset_ds=False): # Recreated on access self._cfg = None @@ -246,7 +252,7 @@ class Init(object): self.paths, copy.deepcopy(self.ds_deps), cfg_list, - pkg_list) + pkg_list, self.reporter) LOG.info("Loaded datasource %s - %s", dsname, ds) self.datasource = ds # Ensure we adjust our path members datasource diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index f4011a79..5700118f 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -32,10 +32,10 @@ class TestReportStartEvent(TestCase): class TestReportFinishEvent(TestCase): - def _report_finish_event(self, successful=None): + def _report_finish_event(self, result=None): event_name, event_description = 'my_test_event', 'my description' reporting.report_finish_event( - event_name, event_description, successful=successful) + event_name, event_description, result=result) return event_name, event_description def assertHandlersPassedObjectWithAsString( @@ -51,7 +51,7 @@ class TestReportFinishEvent(TestCase): self, instantiated_handler_registry): event_name, event_description = self._report_finish_event() expected_string_representation = ': '.join( - ['finish', event_name, event_description]) + ['finish', event_name, reporting.status.SUCCESS, event_description]) self.assertHandlersPassedObjectWithAsString( instantiated_handler_registry.registered_items, expected_string_representation) @@ -61,9 +61,9 @@ class TestReportFinishEvent(TestCase): def test_reporting_successful_finish_has_sensible_string_repr( self, instantiated_handler_registry): event_name, event_description = self._report_finish_event( - successful=True) + result=reporting.status.SUCCESS) expected_string_representation = ': '.join( - ['finish', event_name, 'success', event_description]) + ['finish', event_name, reporting.status.SUCCESS, event_description]) self.assertHandlersPassedObjectWithAsString( instantiated_handler_registry.registered_items, expected_string_representation) @@ -73,9 +73,9 @@ class TestReportFinishEvent(TestCase): def test_reporting_unsuccessful_finish_has_sensible_string_repr( self, instantiated_handler_registry): event_name, event_description = self._report_finish_event( - successful=False) + result=reporting.status.FAIL) expected_string_representation = ': '.join( - ['finish', event_name, 'fail', event_description]) + ['finish', event_name, reporting.status.FAIL, event_description]) self.assertHandlersPassedObjectWithAsString( instantiated_handler_registry.registered_items, expected_string_representation) -- cgit v1.2.3 From 6f174b41496f133af92fb373f3b718eabdebfa05 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 30 Jul 2015 22:22:24 -0400 Subject: fix arg ordering --- bin/cloud-init | 2 +- cloudinit/stages.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/bin/cloud-init b/bin/cloud-init index b0396cdc..6a47e5e8 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -172,7 +172,7 @@ def main_init(name, args): w_msg = welcome_format(name) else: w_msg = welcome_format("%s-local" % (name)) - init = stages.Init(deps) + init = stages.Init(ds_deps=deps, reporter=args.reporter) # Stage 1 init.read_cfg(extract_fns(args)) # Stage 2 diff --git a/cloudinit/stages.py b/cloudinit/stages.py index dbcdbece..2bf7a1c4 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -54,7 +54,7 @@ NULL_DATA_SOURCE = None class Init(object): - def __init__(self, reporter=None, ds_deps=None): + def __init__(self, ds_deps=None, reporter=None): if ds_deps is not None: self.ds_deps = ds_deps else: -- cgit v1.2.3 From f36706442b4c1913ea8f7953993b9e03f3adf623 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 31 Jul 2015 16:12:37 +0000 Subject: address Daniel's comments in review --- bin/cloud-init | 3 ++- cloudinit/reporting.py | 34 +++++++++++++++------------------- cloudinit/sources/__init__.py | 7 +++---- cloudinit/stages.py | 3 ++- 4 files changed, 22 insertions(+), 25 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/bin/cloud-init b/bin/cloud-init index c808eda5..d0ac4c7f 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -636,7 +636,8 @@ def main(): "running single module %s" % args.name) report_on = args.report - args.reporter = reporting.ReportStack(rname, rdesc, reporting=report_on) + args.reporter = reporting.ReportStack( + rname, rdesc, reporting_enabled=report_on) with args.reporter: return util.log_time( logfunc=LOG.debug, msg="cloud-init mode '%s'" % name, diff --git a/cloudinit/reporting.py b/cloudinit/reporting.py index 1bd7df0d..154f4e03 100644 --- a/cloudinit/reporting.py +++ b/cloudinit/reporting.py @@ -11,6 +11,7 @@ report events in a structured manner. import abc import logging +import sys from cloudinit.registry import DictRegistry @@ -83,9 +84,9 @@ class LogHandler(ReportingHandler): logger.info(event.as_string()) -class PrintHandler(ReportingHandler): +class StderrHandler(ReportingHandler): def publish_event(self, event): - print(event.as_string()) + sys.stderr.write(event.as_string() + "\n") def add_configuration(config): @@ -134,23 +135,20 @@ def report_start_event(event_name, event_description): class ReportStack(object): - def __init__(self, name, description, parent=None, reporting=None, - exc_result=None): + def __init__(self, name, description, parent=None, + reporting_enabled=None, result_on_exception=status.FAIL): self.parent = parent self.name = name self.description = description - - if exc_result is None: - exc_result = status.FAIL - self.exc_result = exc_result + self.result_on_exception = result_on_exception # use parents reporting value if not provided - if reporting is None: + if reporting_enabled is None: if parent: - reporting = parent.reporting + reporting_enabled = parent.reporting_enabled else: - reporting = True - self.reporting = reporting + reporting_enabled = True + self.reporting_enabled = reporting_enabled if parent: self.fullname = '/'.join((parent.fullname, name,)) @@ -159,11 +157,10 @@ class ReportStack(object): self.children = {} def __repr__(self): - return ("%s reporting=%s" % (self.fullname, self.reporting)) + return ("%s reporting=%s" % (self.fullname, self.reporting_enabled)) def __enter__(self): - self.exception = None - if self.reporting: + if self.reporting_enabled: report_start_event(self.fullname, self.description) if self.parent: self.parent.children[self.name] = (None, None) @@ -184,18 +181,17 @@ class ReportStack(object): # return tuple of description, and value if exc: # by default, exceptions are fatal - return (self.exc_result, self.description) + return (self.result_on_exception, self.description) return self.childrens_finish_info() def __exit__(self, exc_type, exc_value, traceback): - self.exception = exc_value (result, msg) = self.finish_info(exc_value) if self.parent: self.parent.children[self.name] = (result, msg) - if self.reporting: + if self.reporting_enabled: report_finish_event(self.fullname, msg, result) available_handlers.register_item('log', LogHandler) -available_handlers.register_item('print', PrintHandler) +available_handlers.register_item('print', StderrHandler) add_configuration(DEFAULT_CONFIG) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index c174a58f..0dc75f9e 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -257,7 +257,7 @@ class SearchReportStack(reporting.ReportStack): self.mode, self.source) super(SearchReportStack, self).__init__( name=name, description=description, parent=parent, - exc_result=reporting.status.WARN) + result_on_exception=reporting.status.WARN) def finish_info(self, exc): # return tuple of description, and value @@ -276,10 +276,9 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter): ds_names = [type_utils.obj_name(f) for f in ds_list] LOG.debug("Searching for data source in: %s", ds_names) - for i, cls in enumerate(ds_list): - srcname=ds_names[i] + for name, cls in zip(ds_names, ds_list): try: - with SearchReportStack(srcname, ds_deps, reporter) as rep: + with SearchReportStack(name, ds_deps, reporter) as rep: LOG.debug("Seeing if we can get any data from %s", cls) s = cls(sys_cfg, distro, paths) if s.get_data(): diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 2bf7a1c4..82197d02 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -68,7 +68,8 @@ class Init(object): if reporter is None: reporter = reporting.ReportStack( - name="init-reporter", description="init-desc", reporting=False) + name="init-reporter", description="init-desc", + reporting_enabled=False) self.reporter = reporter def _reset(self, reset_ds=False): -- cgit v1.2.3 From a0c8ba1c53e2834c0d9be8df9b514df0a631e09d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 31 Jul 2015 17:08:50 +0000 Subject: adjust searching so cache hits are logged --- cloudinit/stages.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 82197d02..79d22538 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -241,9 +241,16 @@ class Init(object): def _get_data_source(self): if self.datasource is not NULL_DATA_SOURCE: return self.datasource - ds = self._restore_from_cache() - if ds: - LOG.debug("Restored from cache, datasource: %s", ds) + + with reporting.ReportStack( + name="check-cache", description="attempting to read from cache", + parent=self.reporter) as myrep: + ds = self._restore_from_cache() + if ds: + LOG.debug("Restored from cache, datasource: %s", ds) + myrep.description = "restored from cache" + else: + myrep.description = "no cache found" if not ds: (cfg_list, pkg_list) = self._get_datasources() # Deep copy so that user-data handlers can not modify -- cgit v1.2.3 From 07b452e166b5d2ff34d5558b1dbba42ab0f1f23c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 31 Jul 2015 19:27:52 +0000 Subject: plumb the rest the reporting through --- bin/cloud-init | 9 +++++---- cloudinit/cloud.py | 8 +++++++- cloudinit/reporting.py | 32 +++++++++++++++++++++----------- cloudinit/sources/__init__.py | 32 +++++++------------------------- cloudinit/stages.py | 29 ++++++++++++++++++++++++----- 5 files changed, 64 insertions(+), 46 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/bin/cloud-init b/bin/cloud-init index de3b9fbf..d369a806 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -284,7 +284,7 @@ def main_init(name, args): return (init.datasource, ["Consuming user data failed!"]) # Stage 8 - re-read and apply relevant cloud-config to include user-data - mods = stages.Modules(init, extract_fns(args)) + mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) # Stage 9 try: outfmt_orig = outfmt @@ -329,7 +329,7 @@ def main_modules(action_name, args): if not args.force: return [(msg)] # Stage 3 - mods = stages.Modules(init, extract_fns(args)) + mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) # Stage 4 try: LOG.debug("Closing stdin") @@ -384,7 +384,7 @@ def main_single(name, args): if not args.force: return 1 # Stage 3 - mods = stages.Modules(init, extract_fns(args)) + mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) mod_args = args.module_args if mod_args: LOG.debug("Using passed in arguments %s", mod_args) @@ -630,7 +630,8 @@ def main(): else: rname, rdesc = ("init-network", "searching for network datasources") elif name == "modules": - rname, rdesc = ("modules-%s" % args.mode, "running modules for %s") + rname, rdesc = ("modules-%s" % args.mode, + "running modules for %s" % args.mode) elif name == "single": rname, rdesc = ("single/%s" % args.name, "running single module %s" % args.name) diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index 95e0cfb2..71eb80eb 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -40,12 +40,18 @@ LOG = logging.getLogger(__name__) class Cloud(object): - def __init__(self, datasource, paths, cfg, distro, runners): + def __init__(self, datasource, paths, cfg, distro, runners, reporter=None): self.datasource = datasource self.paths = paths self.distro = distro self._cfg = cfg self._runners = runners + if reporter is None: + reporter = reporting.ReportStack( + name="unnamed-cloud-reporter", + description="unnamed-cloud-reporter", + reporting_enabled=False) + self.reporter = reporter # If a 'user' manipulates logging or logging services # it is typically useful to cause the logging to be diff --git a/cloudinit/reporting.py b/cloudinit/reporting.py index 154f4e03..08014c70 100644 --- a/cloudinit/reporting.py +++ b/cloudinit/reporting.py @@ -86,7 +86,8 @@ class LogHandler(ReportingHandler): class StderrHandler(ReportingHandler): def publish_event(self, event): - sys.stderr.write(event.as_string() + "\n") + #sys.stderr.write(event.as_string() + "\n") + print(event.as_string()) def add_configuration(config): @@ -135,12 +136,14 @@ def report_start_event(event_name, event_description): class ReportStack(object): - def __init__(self, name, description, parent=None, + def __init__(self, name, description, message=None, parent=None, reporting_enabled=None, result_on_exception=status.FAIL): self.parent = parent self.name = name self.description = description + self.message = message self.result_on_exception = result_on_exception + self.result = None # use parents reporting value if not provided if reporting_enabled is None: @@ -160,28 +163,35 @@ class ReportStack(object): return ("%s reporting=%s" % (self.fullname, self.reporting_enabled)) def __enter__(self): + self.result = None if self.reporting_enabled: report_start_event(self.fullname, self.description) if self.parent: self.parent.children[self.name] = (None, None) return self - def childrens_finish_info(self, result=None, description=None): + def childrens_finish_info(self): for cand_result in (status.FAIL, status.WARN): for name, (value, msg) in self.children.items(): if value == cand_result: return (value, "[" + name + "]" + msg) - if result is None: - result = status.SUCCESS - if description is None: - description = self.description - return (result, description) - + return (self.result, self.message) + + @property + def message(self): + if self._message is not None: + return self._message + return self.description + + @message.setter + def message(self, value): + self._message = value + + def finish_info(self, exc): # return tuple of description, and value if exc: - # by default, exceptions are fatal - return (self.result_on_exception, self.description) + return (self.result_on_exception, self.message) return self.childrens_finish_info() def __exit__(self, exc_type, exc_value, traceback): diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 6f2d2276..3b48f173 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -247,29 +247,6 @@ def normalize_pubkey_data(pubkey_data): return keys -class SearchReportStack(reporting.ReportStack): - def __init__(self, source, mode, parent): - self.source = source.replace("DataSource", "") - name = "check-%s" % self.source - self.found = False - self.mode = mode - description = "searching for %s data from %s" % (mode, self.source) - super(SearchReportStack, self).__init__( - name=name, description=description, parent=parent, - result_on_exception=reporting.status.WARN) - - def finish_info(self, exc): - # return tuple of description, and value - if exc: - # by default, exceptions are fatal - return (self.exc_result, self.description) - if self.found: - description = "found %s data from %s" % (self.mode, self.source) - else: - description = "no %s data found from %s" % (self.mode, self.source) - return self.childrens_finish_info(description=description) - - def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter): ds_list = list_sources(cfg_list, ds_deps, pkg_list) ds_names = [type_utils.obj_name(f) for f in ds_list] @@ -277,12 +254,17 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter): LOG.debug("Searching for %s data source in: %s", mode, ds_names) for name, cls in zip(ds_names, ds_list): + myrep = reporting.ReportStack( + name="search-%s-%s" % (mode, name.replace("DataSource", "")), + description="searching for %s data from %s" % (mode, name), + message = "no %s data found from %s" % (mode, name), + parent=reporter) try: - with SearchReportStack(name, mode, reporter) as rep: + with myrep: LOG.debug("Seeing if we can get any data from %s", cls) s = cls(sys_cfg, distro, paths) if s.get_data(): - rep.found = True + myrep.message = "found %s data from %s" % (mode, name) return (s, type_utils.obj_name(cls)) except Exception: util.logexc(LOG, "Getting data from %s failed", cls) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 79d22538..8c79ae4e 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -341,7 +341,8 @@ class Init(object): # Form the needed options to cloudify our members return cloud.Cloud(self.datasource, self.paths, self.cfg, - self.distro, helpers.Runners(self.paths)) + self.distro, helpers.Runners(self.paths), + reporter=self.reporter) def update(self): if not self._write_to_cache(): @@ -507,8 +508,14 @@ class Init(object): def consume_data(self, frequency=PER_INSTANCE): # Consume the userdata first, because we need want to let the part # handlers run first (for merging stuff) - self._consume_userdata(frequency) - self._consume_vendordata(frequency) + with reporting.ReportStack( + "consume-user-data", "reading and applying user-data", + parent=self.reporter): + self._consume_userdata(frequency) + with reporting.ReportStack( + "consume-vendor-data", "reading and applying vendor-data", + parent=self.reporter): + self._consume_userdata(frequency) # Perform post-consumption adjustments so that # modules that run during the init stage reflect @@ -581,11 +588,12 @@ class Init(object): class Modules(object): - def __init__(self, init, cfg_files=None): + def __init__(self, init, cfg_files=None, reporter=None): self.init = init self.cfg_files = cfg_files # Created on first use self._cached_cfg = None + self.reporter = reporter @property def cfg(self): @@ -695,7 +703,18 @@ class Modules(object): which_ran.append(name) # This name will affect the semaphore name created run_name = "config-%s" % (name) - cc.run(run_name, mod.handle, func_args, freq=freq) + + desc="running %s with frequency %s" % (run_name, freq) + myrep = reporting.ReportStack( + name=run_name, description=desc, parent=self.reporter) + + with myrep: + ran, _r = cc.run(run_name, mod.handle, func_args, freq=freq) + if ran: + myrep.message = "%s ran successfully" % run_name + else: + myrep.message = "%s previously ran" % run_name + except Exception as e: util.logexc(LOG, "Running module %s (%s) failed", name, mod) failures.append((name, e)) -- cgit v1.2.3 From 89c564a6fd5ac89869f83541370557e3fa58495c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 2 Aug 2015 17:51:40 -0400 Subject: fix tests from sync change ReportStack to ReportEventStack change default ReportEventStack to be status.SUCCESS instead of None --- bin/cloud-init | 3 +- cloudinit/cloud.py | 2 +- cloudinit/reporting/__init__.py | 91 +++++++++++++++++++++++++++++++++++---- cloudinit/reporting/handlers.py | 7 +++ cloudinit/sources/__init__.py | 2 +- cloudinit/stages.py | 10 ++--- tests/unittests/test_reporting.py | 19 ++++---- 7 files changed, 109 insertions(+), 25 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/bin/cloud-init b/bin/cloud-init index d369a806..51253c42 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -637,7 +637,8 @@ def main(): "running single module %s" % args.name) report_on = args.report - args.reporter = reporting.ReportStack( + reporting.add_configuration({'print': {'type': 'print'}}) + args.reporter = reporting.ReportEventStack( rname, rdesc, reporting_enabled=report_on) with args.reporter: return util.log_time( diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index 71eb80eb..a0fb42a3 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -47,7 +47,7 @@ class Cloud(object): self._cfg = cfg self._runners = runners if reporter is None: - reporter = reporting.ReportStack( + reporter = reporting.ReportEventStack( name="unnamed-cloud-reporter", description="unnamed-cloud-reporter", reporting_enabled=False) diff --git a/cloudinit/reporting/__init__.py b/cloudinit/reporting/__init__.py index b0364eec..78dde715 100644 --- a/cloudinit/reporting/__init__.py +++ b/cloudinit/reporting/__init__.py @@ -22,6 +22,15 @@ DEFAULT_CONFIG = { instantiated_handler_registry = DictRegistry() +class _nameset(set): + def __getattr__(self, name): + if name in self: + return name + raise AttributeError("%s not a valid value" % name) + + +status = _nameset(("SUCCESS", "WARN", "FAIL")) + class ReportingEvent(object): """Encapsulation of event formatting.""" @@ -39,17 +48,16 @@ class ReportingEvent(object): class FinishReportingEvent(ReportingEvent): - def __init__(self, name, description, successful=None): + def __init__(self, name, description, result=status.SUCCESS): super(FinishReportingEvent, self).__init__( FINISH_EVENT_TYPE, name, description) - self.successful = successful + self.result = result + if result not in status: + raise ValueError("Invalid result: %s" % result) def as_string(self): - if self.successful is None: - return super(FinishReportingEvent, self).as_string() - success_string = 'success' if self.successful else 'fail' return '{0}: {1}: {2}: {3}'.format( - self.event_type, self.name, success_string, self.description) + self.event_type, self.name, self.result, self.description) def add_configuration(config): @@ -74,12 +82,13 @@ def report_event(event): handler.publish_event(event) -def report_finish_event(event_name, event_description, successful=None): +def report_finish_event(event_name, event_description, + result=status.SUCCESS): """Report a "finish" event. See :py:func:`.report_event` for parameter details. """ - event = FinishReportingEvent(event_name, event_description, successful) + event = FinishReportingEvent(event_name, event_description, result) return report_event(event) @@ -97,4 +106,70 @@ def report_start_event(event_name, event_description): return report_event(event) +class ReportEventStack(object): + def __init__(self, name, description, message=None, parent=None, + reporting_enabled=None, result_on_exception=status.FAIL): + self.parent = parent + self.name = name + self.description = description + self.message = message + self.result_on_exception = result_on_exception + self.result = status.SUCCESS + + # use parents reporting value if not provided + if reporting_enabled is None: + if parent: + reporting_enabled = parent.reporting_enabled + else: + reporting_enabled = True + self.reporting_enabled = reporting_enabled + + if parent: + self.fullname = '/'.join((parent.fullname, name,)) + else: + self.fullname = self.name + self.children = {} + + def __repr__(self): + return ("%s reporting=%s" % (self.fullname, self.reporting_enabled)) + + def __enter__(self): + self.result = status.SUCCESS + if self.reporting_enabled: + report_start_event(self.fullname, self.description) + if self.parent: + self.parent.children[self.name] = (None, None) + return self + + def childrens_finish_info(self): + for cand_result in (status.FAIL, status.WARN): + for name, (value, msg) in self.children.items(): + if value == cand_result: + return (value, "[" + name + "]" + msg) + return (self.result, self.message) + + @property + def message(self): + if self._message is not None: + return self._message + return self.description + + @message.setter + def message(self, value): + self._message = value + + def finish_info(self, exc): + # return tuple of description, and value + if exc: + return (self.result_on_exception, self.message) + return self.childrens_finish_info() + + def __exit__(self, exc_type, exc_value, traceback): + (result, msg) = self.finish_info(exc_value) + if self.parent: + self.parent.children[self.name] = (result, msg) + if self.reporting_enabled: + report_finish_event(self.fullname, msg, result) + + add_configuration(DEFAULT_CONFIG) diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index be323f53..1d5ca524 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -21,5 +21,12 @@ class LogHandler(ReportingHandler): logger.info(event.as_string()) +class StderrHandler(ReportingHandler): + def publish_event(self, event): + #sys.stderr.write(event.as_string() + "\n") + print(event.as_string()) + + available_handlers = DictRegistry() available_handlers.register_item('log', LogHandler) +available_handlers.register_item('print', StderrHandler) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 3b48f173..d07cf1fa 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -254,7 +254,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter): LOG.debug("Searching for %s data source in: %s", mode, ds_names) for name, cls in zip(ds_names, ds_list): - myrep = reporting.ReportStack( + myrep = reporting.ReportEventStack( name="search-%s-%s" % (mode, name.replace("DataSource", "")), description="searching for %s data from %s" % (mode, name), message = "no %s data found from %s" % (mode, name), diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 8c79ae4e..42989bb4 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -67,7 +67,7 @@ class Init(object): self.datasource = NULL_DATA_SOURCE if reporter is None: - reporter = reporting.ReportStack( + reporter = reporting.ReportEventStack( name="init-reporter", description="init-desc", reporting_enabled=False) self.reporter = reporter @@ -242,7 +242,7 @@ class Init(object): if self.datasource is not NULL_DATA_SOURCE: return self.datasource - with reporting.ReportStack( + with reporting.ReportEventStack( name="check-cache", description="attempting to read from cache", parent=self.reporter) as myrep: ds = self._restore_from_cache() @@ -508,11 +508,11 @@ class Init(object): def consume_data(self, frequency=PER_INSTANCE): # Consume the userdata first, because we need want to let the part # handlers run first (for merging stuff) - with reporting.ReportStack( + with reporting.ReportEventStack( "consume-user-data", "reading and applying user-data", parent=self.reporter): self._consume_userdata(frequency) - with reporting.ReportStack( + with reporting.ReportEventStack( "consume-vendor-data", "reading and applying vendor-data", parent=self.reporter): self._consume_userdata(frequency) @@ -705,7 +705,7 @@ class Modules(object): run_name = "config-%s" % (name) desc="running %s with frequency %s" % (run_name, freq) - myrep = reporting.ReportStack( + myrep = reporting.ReportEventStack( name=run_name, description=desc, parent=self.reporter) with myrep: diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 5700118f..4f4cf3a4 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -32,7 +32,7 @@ class TestReportStartEvent(TestCase): class TestReportFinishEvent(TestCase): - def _report_finish_event(self, result=None): + def _report_finish_event(self, result=reporting.status.SUCCESS): event_name, event_description = 'my_test_event', 'my description' reporting.report_finish_event( event_name, event_description, result=result) @@ -95,31 +95,32 @@ class TestReportingHandler(TestCase): def test_no_default_publish_event_implementation(self): self.assertRaises(NotImplementedError, - reporting.ReportingHandler().publish_event, None) + reporting.handlers.ReportingHandler().publish_event, + None) class TestLogHandler(TestCase): - @mock.patch.object(reporting.logging, 'getLogger') + @mock.patch.object(reporting.handlers.logging, 'getLogger') def test_appropriate_logger_used(self, getLogger): event_type, event_name = 'test_type', 'test_name' event = reporting.ReportingEvent(event_type, event_name, 'description') - reporting.LogHandler().publish_event(event) + reporting.handlers.LogHandler().publish_event(event) self.assertEqual( [mock.call( 'cloudinit.reporting.{0}.{1}'.format(event_type, event_name))], getLogger.call_args_list) - @mock.patch.object(reporting.logging, 'getLogger') + @mock.patch.object(reporting.handlers.logging, 'getLogger') def test_single_log_message_at_info_published(self, getLogger): event = reporting.ReportingEvent('type', 'name', 'description') - reporting.LogHandler().publish_event(event) + reporting.handlers.LogHandler().publish_event(event) self.assertEqual(1, getLogger.return_value.info.call_count) - @mock.patch.object(reporting.logging, 'getLogger') + @mock.patch.object(reporting.handlers.logging, 'getLogger') def test_log_message_uses_event_as_string(self, getLogger): event = reporting.ReportingEvent('type', 'name', 'description') - reporting.LogHandler().publish_event(event) + reporting.handlers.LogHandler().publish_event(event) self.assertIn(event.as_string(), getLogger.return_value.info.call_args[0][0]) @@ -130,7 +131,7 @@ class TestDefaultRegisteredHandler(TestCase): registered_items = ( reporting.instantiated_handler_registry.registered_items) for _, item in registered_items.items(): - if isinstance(item, reporting.LogHandler): + if isinstance(item, reporting.handlers.LogHandler): break else: self.fail('No reporting LogHandler registered by default.') -- cgit v1.2.3 From 9f49cf601fd7bedb429d2bfcd7e877c9ed5f3690 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 4 Aug 2015 21:50:08 -0500 Subject: fix all tests (were broken due to copied code call to userdata twice --- cloudinit/stages.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 42989bb4..7b489b9f 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -243,14 +243,14 @@ class Init(object): return self.datasource with reporting.ReportEventStack( - name="check-cache", description="attempting to read from cache", - parent=self.reporter) as myrep: - ds = self._restore_from_cache() - if ds: - LOG.debug("Restored from cache, datasource: %s", ds) - myrep.description = "restored from cache" - else: - myrep.description = "no cache found" + name="check-cache", description="attempting to read from cache", + parent=self.reporter) as myrep: + ds = self._restore_from_cache() + if ds: + LOG.debug("Restored from cache, datasource: %s", ds) + myrep.description = "restored from cache" + else: + myrep.description = "no cache found" if not ds: (cfg_list, pkg_list) = self._get_datasources() # Deep copy so that user-data handlers can not modify @@ -515,7 +515,7 @@ class Init(object): with reporting.ReportEventStack( "consume-vendor-data", "reading and applying vendor-data", parent=self.reporter): - self._consume_userdata(frequency) + self._consume_vendordata(frequency) # Perform post-consumption adjustments so that # modules that run during the init stage reflect @@ -593,6 +593,10 @@ class Modules(object): self.cfg_files = cfg_files # Created on first use self._cached_cfg = None + if reporter is None: + reporter = reporting.ReportEventStack( + name="module-reporter", description="module-desc", + reporting_enabled=False) self.reporter = reporter @property -- cgit v1.2.3 From 5585b397cfb4ba397e9cfba3d86e3d10af20eb71 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 4 Aug 2015 22:01:27 -0500 Subject: fix pep8 --- bin/cloud-init | 3 ++- cloudinit/reporting/handlers.py | 7 ------- cloudinit/sources/__init__.py | 2 +- cloudinit/stages.py | 10 ++++++---- tests/unittests/test_reporting.py | 6 ++++-- 5 files changed, 13 insertions(+), 15 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/bin/cloud-init b/bin/cloud-init index 51253c42..40cdbb06 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -628,7 +628,8 @@ def main(): if args.local: rname, rdesc = ("init-local", "searching for local datasources") else: - rname, rdesc = ("init-network", "searching for network datasources") + rname, rdesc = ("init-network", + "searching for network datasources") elif name == "modules": rname, rdesc = ("modules-%s" % args.mode, "running modules for %s" % args.mode) diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 1d5ca524..be323f53 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -21,12 +21,5 @@ class LogHandler(ReportingHandler): logger.info(event.as_string()) -class StderrHandler(ReportingHandler): - def publish_event(self, event): - #sys.stderr.write(event.as_string() + "\n") - print(event.as_string()) - - available_handlers = DictRegistry() available_handlers.register_item('log', LogHandler) -available_handlers.register_item('print', StderrHandler) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index cf50c1fb..838cd198 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -257,7 +257,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter): myrep = reporting.ReportEventStack( name="search-%s" % name.replace("DataSource", ""), description="searching for %s data from %s" % (mode, name), - message = "no %s data found from %s" % (mode, name), + message="no %s data found from %s" % (mode, name), parent=reporter) try: with myrep: diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 7b489b9f..d300709d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -243,7 +243,8 @@ class Init(object): return self.datasource with reporting.ReportEventStack( - name="check-cache", description="attempting to read from cache", + name="check-cache", + description="attempting to read from cache", parent=self.reporter) as myrep: ds = self._restore_from_cache() if ds: @@ -708,17 +709,18 @@ class Modules(object): # This name will affect the semaphore name created run_name = "config-%s" % (name) - desc="running %s with frequency %s" % (run_name, freq) + desc = "running %s with frequency %s" % (run_name, freq) myrep = reporting.ReportEventStack( name=run_name, description=desc, parent=self.reporter) with myrep: - ran, _r = cc.run(run_name, mod.handle, func_args, freq=freq) + ran, _r = cc.run(run_name, mod.handle, func_args, + freq=freq) if ran: myrep.message = "%s ran successfully" % run_name else: myrep.message = "%s previously ran" % run_name - + except Exception as e: util.logexc(LOG, "Running module %s (%s) failed", name, mod) failures.append((name, e)) diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 4f4cf3a4..ddfac541 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -51,7 +51,8 @@ class TestReportFinishEvent(TestCase): self, instantiated_handler_registry): event_name, event_description = self._report_finish_event() expected_string_representation = ': '.join( - ['finish', event_name, reporting.status.SUCCESS, event_description]) + ['finish', event_name, reporting.status.SUCCESS, + event_description]) self.assertHandlersPassedObjectWithAsString( instantiated_handler_registry.registered_items, expected_string_representation) @@ -63,7 +64,8 @@ class TestReportFinishEvent(TestCase): event_name, event_description = self._report_finish_event( result=reporting.status.SUCCESS) expected_string_representation = ': '.join( - ['finish', event_name, reporting.status.SUCCESS, event_description]) + ['finish', event_name, reporting.status.SUCCESS, + event_description]) self.assertHandlersPassedObjectWithAsString( instantiated_handler_registry.registered_items, expected_string_representation) -- cgit v1.2.3 From 50bcb0f77d29a76a03946c6da13b15be25257402 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 31 Aug 2015 13:33:30 -0400 Subject: split 'events' portion of reporting into separate file this just separates events from other things that could conceivably be reported. --- cloudinit/cloud.py | 4 +- cloudinit/reporting/__init__.py | 203 +----------------------------------- cloudinit/reporting/events.py | 210 ++++++++++++++++++++++++++++++++++++++ cloudinit/sources/__init__.py | 4 +- cloudinit/stages.py | 14 +-- tests/unittests/test_reporting.py | 121 +++++++++++----------- 6 files changed, 285 insertions(+), 271 deletions(-) create mode 100644 cloudinit/reporting/events.py (limited to 'cloudinit/stages.py') diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index edee3887..3e6be203 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -24,7 +24,7 @@ import copy import os from cloudinit import log as logging -from cloudinit import reporting +from cloudinit.reporting import events LOG = logging.getLogger(__name__) @@ -48,7 +48,7 @@ class Cloud(object): self._cfg = cfg self._runners = runners if reporter is None: - reporter = reporting.ReportEventStack( + reporter = events.ReportEventStack( name="unnamed-cloud-reporter", description="unnamed-cloud-reporter", reporting_enabled=False) diff --git a/cloudinit/reporting/__init__.py b/cloudinit/reporting/__init__.py index 502af95c..6b41ae61 100644 --- a/cloudinit/reporting/__init__.py +++ b/cloudinit/reporting/__init__.py @@ -1,7 +1,6 @@ # Copyright 2015 Canonical Ltd. # This file is part of cloud-init. See LICENCE file for license information. # -# vi: ts=4 expandtab """ cloud-init reporting framework @@ -10,66 +9,13 @@ report events in a structured manner. """ from ..registry import DictRegistry -from ..reporting.handlers import available_handlers - - -FINISH_EVENT_TYPE = 'finish' -START_EVENT_TYPE = 'start' +from .handlers import available_handlers DEFAULT_CONFIG = { 'logging': {'type': 'log'}, } -class _nameset(set): - def __getattr__(self, name): - if name in self: - return name - raise AttributeError("%s not a valid value" % name) - - -status = _nameset(("SUCCESS", "WARN", "FAIL")) - - -class ReportingEvent(object): - """Encapsulation of event formatting.""" - - def __init__(self, event_type, name, description): - self.event_type = event_type - self.name = name - self.description = description - - def as_string(self): - """The event represented as a string.""" - return '{0}: {1}: {2}'.format( - self.event_type, self.name, self.description) - - def as_dict(self): - """The event represented as a dictionary.""" - return {'name': self.name, 'description': self.description, - 'event_type': self.event_type} - - -class FinishReportingEvent(ReportingEvent): - - def __init__(self, name, description, result=status.SUCCESS): - super(FinishReportingEvent, self).__init__( - FINISH_EVENT_TYPE, name, description) - self.result = result - if result not in status: - raise ValueError("Invalid result: %s" % result) - - def as_string(self): - return '{0}: {1}: {2}: {3}'.format( - self.event_type, self.name, self.result, self.description) - - def as_dict(self): - """The event represented as json friendly.""" - data = super(FinishReportingEvent, self).as_dict() - data['result'] = self.result - return data - - def update_configuration(config): """Update the instanciated_handler_registry. @@ -90,150 +36,7 @@ def update_configuration(config): instantiated_handler_registry.register_item(handler_name, instance) -def report_event(event): - """Report an event to all registered event handlers. - - This should generally be called via one of the other functions in - the reporting module. - - :param event_type: - The type of the event; this should be a constant from the - reporting module. - """ - for _, handler in instantiated_handler_registry.registered_items.items(): - handler.publish_event(event) - - -def report_finish_event(event_name, event_description, - result=status.SUCCESS): - """Report a "finish" event. - - See :py:func:`.report_event` for parameter details. - """ - event = FinishReportingEvent(event_name, event_description, result) - return report_event(event) - - -def report_start_event(event_name, event_description): - """Report a "start" event. - - :param event_name: - The name of the event; this should be a topic which events would - share (e.g. it will be the same for start and finish events). - - :param event_description: - A human-readable description of the event that has occurred. - """ - event = ReportingEvent(START_EVENT_TYPE, event_name, event_description) - return report_event(event) - - -class ReportEventStack(object): - """Context Manager for using :py:func:`report_event` - - This enables calling :py:func:`report_start_event` and - :py:func:`report_finish_event` through a context manager. - - :param name: - the name of the event - - :param description: - the event's description, passed on to :py:func:`report_start_event` - - :param message: - the description to use for the finish event. defaults to - :param:description. - - :param parent: - :type parent: :py:class:ReportEventStack or None - The parent of this event. The parent is populated with - results of all its children. The name used in reporting - is / - - :param reporting_enabled: - Indicates if reporting events should be generated. - If not provided, defaults to the parent's value, or True if no parent - is provided. - - :param result_on_exception: - The result value to set if an exception is caught. default - value is FAIL. - """ - def __init__(self, name, description, message=None, parent=None, - reporting_enabled=None, result_on_exception=status.FAIL): - self.parent = parent - self.name = name - self.description = description - self.message = message - self.result_on_exception = result_on_exception - self.result = status.SUCCESS - - # use parents reporting value if not provided - if reporting_enabled is None: - if parent: - reporting_enabled = parent.reporting_enabled - else: - reporting_enabled = True - self.reporting_enabled = reporting_enabled - - if parent: - self.fullname = '/'.join((parent.fullname, name,)) - else: - self.fullname = self.name - self.children = {} - - def __repr__(self): - return ("ReportEventStack(%s, %s, reporting_enabled=%s)" % - (self.name, self.description, self.reporting_enabled)) - - def __enter__(self): - self.result = status.SUCCESS - if self.reporting_enabled: - report_start_event(self.fullname, self.description) - if self.parent: - self.parent.children[self.name] = (None, None) - return self - - def _childrens_finish_info(self): - for cand_result in (status.FAIL, status.WARN): - for name, (value, msg) in self.children.items(): - if value == cand_result: - return (value, self.message) - return (self.result, self.message) - - @property - def result(self): - return self._result - - @result.setter - def result(self, value): - if value not in status: - raise ValueError("'%s' not a valid result" % value) - self._result = value - - @property - def message(self): - if self._message is not None: - return self._message - return self.description - - @message.setter - def message(self, value): - self._message = value - - def _finish_info(self, exc): - # return tuple of description, and value - if exc: - return (self.result_on_exception, self.message) - return self._childrens_finish_info() - - def __exit__(self, exc_type, exc_value, traceback): - (result, msg) = self._finish_info(exc_value) - if self.parent: - self.parent.children[self.name] = (result, msg) - if self.reporting_enabled: - report_finish_event(self.fullname, msg, result) - - instantiated_handler_registry = DictRegistry() update_configuration(DEFAULT_CONFIG) + +# vi: ts=4 expandtab diff --git a/cloudinit/reporting/events.py b/cloudinit/reporting/events.py new file mode 100644 index 00000000..e35e41dd --- /dev/null +++ b/cloudinit/reporting/events.py @@ -0,0 +1,210 @@ +# Copyright 2015 Canonical Ltd. +# This file is part of cloud-init. See LICENCE file for license information. +# +""" +cloud-init events + +Report events in a structured manner. +The events here are most likely used via reporting. +""" + +from . import instantiated_handler_registry + +FINISH_EVENT_TYPE = 'finish' +START_EVENT_TYPE = 'start' + + +class _nameset(set): + def __getattr__(self, name): + if name in self: + return name + raise AttributeError("%s not a valid value" % name) + + +status = _nameset(("SUCCESS", "WARN", "FAIL")) + + +class ReportingEvent(object): + """Encapsulation of event formatting.""" + + def __init__(self, event_type, name, description): + self.event_type = event_type + self.name = name + self.description = description + + def as_string(self): + """The event represented as a string.""" + return '{0}: {1}: {2}'.format( + self.event_type, self.name, self.description) + + def as_dict(self): + """The event represented as a dictionary.""" + return {'name': self.name, 'description': self.description, + 'event_type': self.event_type} + + +class FinishReportingEvent(ReportingEvent): + + def __init__(self, name, description, result=status.SUCCESS): + super(FinishReportingEvent, self).__init__( + FINISH_EVENT_TYPE, name, description) + self.result = result + if result not in status: + raise ValueError("Invalid result: %s" % result) + + def as_string(self): + return '{0}: {1}: {2}: {3}'.format( + self.event_type, self.name, self.result, self.description) + + def as_dict(self): + """The event represented as json friendly.""" + data = super(FinishReportingEvent, self).as_dict() + data['result'] = self.result + return data + + +def report_event(event): + """Report an event to all registered event handlers. + + This should generally be called via one of the other functions in + the reporting module. + + :param event_type: + The type of the event; this should be a constant from the + reporting module. + """ + for _, handler in instantiated_handler_registry.registered_items.items(): + handler.publish_event(event) + + +def report_finish_event(event_name, event_description, + result=status.SUCCESS): + """Report a "finish" event. + + See :py:func:`.report_event` for parameter details. + """ + event = FinishReportingEvent(event_name, event_description, result) + return report_event(event) + + +def report_start_event(event_name, event_description): + """Report a "start" event. + + :param event_name: + The name of the event; this should be a topic which events would + share (e.g. it will be the same for start and finish events). + + :param event_description: + A human-readable description of the event that has occurred. + """ + event = ReportingEvent(START_EVENT_TYPE, event_name, event_description) + return report_event(event) + + +class ReportEventStack(object): + """Context Manager for using :py:func:`report_event` + + This enables calling :py:func:`report_start_event` and + :py:func:`report_finish_event` through a context manager. + + :param name: + the name of the event + + :param description: + the event's description, passed on to :py:func:`report_start_event` + + :param message: + the description to use for the finish event. defaults to + :param:description. + + :param parent: + :type parent: :py:class:ReportEventStack or None + The parent of this event. The parent is populated with + results of all its children. The name used in reporting + is / + + :param reporting_enabled: + Indicates if reporting events should be generated. + If not provided, defaults to the parent's value, or True if no parent + is provided. + + :param result_on_exception: + The result value to set if an exception is caught. default + value is FAIL. + """ + def __init__(self, name, description, message=None, parent=None, + reporting_enabled=None, result_on_exception=status.FAIL): + self.parent = parent + self.name = name + self.description = description + self.message = message + self.result_on_exception = result_on_exception + self.result = status.SUCCESS + + # use parents reporting value if not provided + if reporting_enabled is None: + if parent: + reporting_enabled = parent.reporting_enabled + else: + reporting_enabled = True + self.reporting_enabled = reporting_enabled + + if parent: + self.fullname = '/'.join((parent.fullname, name,)) + else: + self.fullname = self.name + self.children = {} + + def __repr__(self): + return ("ReportEventStack(%s, %s, reporting_enabled=%s)" % + (self.name, self.description, self.reporting_enabled)) + + def __enter__(self): + self.result = status.SUCCESS + if self.reporting_enabled: + report_start_event(self.fullname, self.description) + if self.parent: + self.parent.children[self.name] = (None, None) + return self + + def _childrens_finish_info(self): + for cand_result in (status.FAIL, status.WARN): + for name, (value, msg) in self.children.items(): + if value == cand_result: + return (value, self.message) + return (self.result, self.message) + + @property + def result(self): + return self._result + + @result.setter + def result(self, value): + if value not in status: + raise ValueError("'%s' not a valid result" % value) + self._result = value + + @property + def message(self): + if self._message is not None: + return self._message + return self.description + + @message.setter + def message(self, value): + self._message = value + + def _finish_info(self, exc): + # return tuple of description, and value + if exc: + return (self.result_on_exception, self.message) + return self._childrens_finish_info() + + def __exit__(self, exc_type, exc_value, traceback): + (result, msg) = self._finish_info(exc_value) + if self.parent: + self.parent.children[self.name] = (result, msg) + if self.reporting_enabled: + report_finish_event(self.fullname, msg, result) + +# vi: ts=4 expandtab diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 838cd198..d3cfa560 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -27,12 +27,12 @@ import six from cloudinit import importer from cloudinit import log as logging -from cloudinit import reporting from cloudinit import type_utils from cloudinit import user_data as ud from cloudinit import util from cloudinit.filters import launch_index +from cloudinit.reporting import events DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" @@ -254,7 +254,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter): LOG.debug("Searching for %s data source in: %s", mode, ds_names) for name, cls in zip(ds_names, ds_list): - myrep = reporting.ReportEventStack( + myrep = events.ReportEventStack( name="search-%s" % name.replace("DataSource", ""), description="searching for %s data from %s" % (mode, name), message="no %s data found from %s" % (mode, name), diff --git a/cloudinit/stages.py b/cloudinit/stages.py index d300709d..9f192c8d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -46,7 +46,7 @@ from cloudinit import log as logging from cloudinit import sources from cloudinit import type_utils from cloudinit import util -from cloudinit import reporting +from cloudinit.reporting import events LOG = logging.getLogger(__name__) @@ -67,7 +67,7 @@ class Init(object): self.datasource = NULL_DATA_SOURCE if reporter is None: - reporter = reporting.ReportEventStack( + reporter = events.ReportEventStack( name="init-reporter", description="init-desc", reporting_enabled=False) self.reporter = reporter @@ -242,7 +242,7 @@ class Init(object): if self.datasource is not NULL_DATA_SOURCE: return self.datasource - with reporting.ReportEventStack( + with events.ReportEventStack( name="check-cache", description="attempting to read from cache", parent=self.reporter) as myrep: @@ -509,11 +509,11 @@ class Init(object): def consume_data(self, frequency=PER_INSTANCE): # Consume the userdata first, because we need want to let the part # handlers run first (for merging stuff) - with reporting.ReportEventStack( + with events.ReportEventStack( "consume-user-data", "reading and applying user-data", parent=self.reporter): self._consume_userdata(frequency) - with reporting.ReportEventStack( + with events.ReportEventStack( "consume-vendor-data", "reading and applying vendor-data", parent=self.reporter): self._consume_vendordata(frequency) @@ -595,7 +595,7 @@ class Modules(object): # Created on first use self._cached_cfg = None if reporter is None: - reporter = reporting.ReportEventStack( + reporter = events.ReportEventStack( name="module-reporter", description="module-desc", reporting_enabled=False) self.reporter = reporter @@ -710,7 +710,7 @@ class Modules(object): run_name = "config-%s" % (name) desc = "running %s with frequency %s" % (run_name, freq) - myrep = reporting.ReportEventStack( + myrep = events.ReportEventStack( name=run_name, description=desc, parent=self.reporter) with myrep: diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 66d4e87e..bb67ef73 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -5,6 +5,7 @@ from cloudinit import reporting from cloudinit.reporting import handlers +from cloudinit.reporting import events from .helpers import (mock, TestCase) @@ -16,12 +17,12 @@ def _fake_registry(): class TestReportStartEvent(TestCase): - @mock.patch('cloudinit.reporting.instantiated_handler_registry', + @mock.patch('cloudinit.reporting.events.instantiated_handler_registry', new_callable=_fake_registry) def test_report_start_event_passes_something_with_as_string_to_handlers( self, instantiated_handler_registry): event_name, event_description = 'my_test_event', 'my description' - reporting.report_start_event(event_name, event_description) + events.report_start_event(event_name, event_description) expected_string_representation = ': '.join( ['start', event_name, event_description]) for _, handler in ( @@ -33,9 +34,9 @@ class TestReportStartEvent(TestCase): class TestReportFinishEvent(TestCase): - def _report_finish_event(self, result=reporting.status.SUCCESS): + def _report_finish_event(self, result=events.status.SUCCESS): event_name, event_description = 'my_test_event', 'my description' - reporting.report_finish_event( + events.report_finish_event( event_name, event_description, result=result) return event_name, event_description @@ -46,39 +47,39 @@ class TestReportFinishEvent(TestCase): event = handler.publish_event.call_args[0][0] self.assertEqual(expected_as_string, event.as_string()) - @mock.patch('cloudinit.reporting.instantiated_handler_registry', + @mock.patch('cloudinit.reporting.events.instantiated_handler_registry', new_callable=_fake_registry) def test_report_finish_event_passes_something_with_as_string_to_handlers( self, instantiated_handler_registry): event_name, event_description = self._report_finish_event() expected_string_representation = ': '.join( - ['finish', event_name, reporting.status.SUCCESS, + ['finish', event_name, events.status.SUCCESS, event_description]) self.assertHandlersPassedObjectWithAsString( instantiated_handler_registry.registered_items, expected_string_representation) - @mock.patch('cloudinit.reporting.instantiated_handler_registry', + @mock.patch('cloudinit.reporting.events.instantiated_handler_registry', new_callable=_fake_registry) def test_reporting_successful_finish_has_sensible_string_repr( self, instantiated_handler_registry): event_name, event_description = self._report_finish_event( - result=reporting.status.SUCCESS) + result=events.status.SUCCESS) expected_string_representation = ': '.join( - ['finish', event_name, reporting.status.SUCCESS, + ['finish', event_name, events.status.SUCCESS, event_description]) self.assertHandlersPassedObjectWithAsString( instantiated_handler_registry.registered_items, expected_string_representation) - @mock.patch('cloudinit.reporting.instantiated_handler_registry', + @mock.patch('cloudinit.reporting.events.instantiated_handler_registry', new_callable=_fake_registry) def test_reporting_unsuccessful_finish_has_sensible_string_repr( self, instantiated_handler_registry): event_name, event_description = self._report_finish_event( - result=reporting.status.FAIL) + result=events.status.FAIL) expected_string_representation = ': '.join( - ['finish', event_name, reporting.status.FAIL, event_description]) + ['finish', event_name, events.status.FAIL, event_description]) self.assertHandlersPassedObjectWithAsString( instantiated_handler_registry.registered_items, expected_string_representation) @@ -91,14 +92,14 @@ class TestReportingEvent(TestCase): def test_as_string(self): event_type, name, description = 'test_type', 'test_name', 'test_desc' - event = reporting.ReportingEvent(event_type, name, description) + event = events.ReportingEvent(event_type, name, description) expected_string_representation = ': '.join( [event_type, name, description]) self.assertEqual(expected_string_representation, event.as_string()) def test_as_dict(self): event_type, name, desc = 'test_type', 'test_name', 'test_desc' - event = reporting.ReportingEvent(event_type, name, desc) + event = events.ReportingEvent(event_type, name, desc) self.assertEqual( {'event_type': event_type, 'name': name, 'description': desc}, event.as_dict()) @@ -106,9 +107,9 @@ class TestReportingEvent(TestCase): class TestFinishReportingEvent(TestCase): def test_as_has_result(self): - result = reporting.status.SUCCESS + result = events.status.SUCCESS name, desc = 'test_name', 'test_desc' - event = reporting.FinishReportingEvent(name, desc, result) + event = events.FinishReportingEvent(name, desc, result) ret = event.as_dict() self.assertTrue('result' in ret) self.assertEqual(ret['result'], result) @@ -126,7 +127,7 @@ class TestLogHandler(TestCase): @mock.patch.object(reporting.handlers.logging, 'getLogger') def test_appropriate_logger_used(self, getLogger): event_type, event_name = 'test_type', 'test_name' - event = reporting.ReportingEvent(event_type, event_name, 'description') + event = events.ReportingEvent(event_type, event_name, 'description') reporting.handlers.LogHandler().publish_event(event) self.assertEqual( [mock.call( @@ -135,13 +136,13 @@ class TestLogHandler(TestCase): @mock.patch.object(reporting.handlers.logging, 'getLogger') def test_single_log_message_at_info_published(self, getLogger): - event = reporting.ReportingEvent('type', 'name', 'description') + event = events.ReportingEvent('type', 'name', 'description') reporting.handlers.LogHandler().publish_event(event) self.assertEqual(1, getLogger.return_value.log.call_count) @mock.patch.object(reporting.handlers.logging, 'getLogger') def test_log_message_uses_event_as_string(self, getLogger): - event = reporting.ReportingEvent('type', 'name', 'description') + event = events.ReportingEvent('type', 'name', 'description') reporting.handlers.LogHandler(level="INFO").publish_event(event) self.assertIn(event.as_string(), getLogger.return_value.log.call_args[0][1]) @@ -232,49 +233,49 @@ class TestReportingConfiguration(TestCase): class TestReportingEventStack(TestCase): - @mock.patch('cloudinit.reporting.report_finish_event') - @mock.patch('cloudinit.reporting.report_start_event') + @mock.patch('cloudinit.reporting.events.report_finish_event') + @mock.patch('cloudinit.reporting.events.report_start_event') def test_start_and_finish_success(self, report_start, report_finish): - with reporting.ReportEventStack(name="myname", description="mydesc"): + with events.ReportEventStack(name="myname", description="mydesc"): pass self.assertEqual( [mock.call('myname', 'mydesc')], report_start.call_args_list) self.assertEqual( - [mock.call('myname', 'mydesc', reporting.status.SUCCESS)], + [mock.call('myname', 'mydesc', events.status.SUCCESS)], report_finish.call_args_list) - @mock.patch('cloudinit.reporting.report_finish_event') - @mock.patch('cloudinit.reporting.report_start_event') + @mock.patch('cloudinit.reporting.events.report_finish_event') + @mock.patch('cloudinit.reporting.events.report_start_event') def test_finish_exception_defaults_fail(self, report_start, report_finish): name = "myname" desc = "mydesc" try: - with reporting.ReportEventStack(name, description=desc): + with events.ReportEventStack(name, description=desc): raise ValueError("This didnt work") except ValueError: pass self.assertEqual([mock.call(name, desc)], report_start.call_args_list) self.assertEqual( - [mock.call(name, desc, reporting.status.FAIL)], + [mock.call(name, desc, events.status.FAIL)], report_finish.call_args_list) - @mock.patch('cloudinit.reporting.report_finish_event') - @mock.patch('cloudinit.reporting.report_start_event') + @mock.patch('cloudinit.reporting.events.report_finish_event') + @mock.patch('cloudinit.reporting.events.report_start_event') def test_result_on_exception_used(self, report_start, report_finish): name = "myname" desc = "mydesc" try: - with reporting.ReportEventStack( - name, desc, result_on_exception=reporting.status.WARN): + with events.ReportEventStack( + name, desc, result_on_exception=events.status.WARN): raise ValueError("This didnt work") except ValueError: pass self.assertEqual([mock.call(name, desc)], report_start.call_args_list) self.assertEqual( - [mock.call(name, desc, reporting.status.WARN)], + [mock.call(name, desc, events.status.WARN)], report_finish.call_args_list) - @mock.patch('cloudinit.reporting.report_start_event') + @mock.patch('cloudinit.reporting.events.report_start_event') def test_child_fullname_respects_parent(self, report_start): parent_name = "topname" c1_name = "c1name" @@ -282,59 +283,59 @@ class TestReportingEventStack(TestCase): c2_expected_fullname = '/'.join([parent_name, c1_name, c2_name]) c1_expected_fullname = '/'.join([parent_name, c1_name]) - parent = reporting.ReportEventStack(parent_name, "topdesc") - c1 = reporting.ReportEventStack(c1_name, "c1desc", parent=parent) - c2 = reporting.ReportEventStack(c2_name, "c2desc", parent=c1) + parent = events.ReportEventStack(parent_name, "topdesc") + c1 = events.ReportEventStack(c1_name, "c1desc", parent=parent) + c2 = events.ReportEventStack(c2_name, "c2desc", parent=c1) with c1: report_start.assert_called_with(c1_expected_fullname, "c1desc") with c2: report_start.assert_called_with(c2_expected_fullname, "c2desc") - @mock.patch('cloudinit.reporting.report_finish_event') - @mock.patch('cloudinit.reporting.report_start_event') + @mock.patch('cloudinit.reporting.events.report_finish_event') + @mock.patch('cloudinit.reporting.events.report_start_event') def test_child_result_bubbles_up(self, report_start, report_finish): - parent = reporting.ReportEventStack("topname", "topdesc") - child = reporting.ReportEventStack("c_name", "c_desc", parent=parent) + parent = events.ReportEventStack("topname", "topdesc") + child = events.ReportEventStack("c_name", "c_desc", parent=parent) with parent: with child: - child.result = reporting.status.WARN + child.result = events.status.WARN report_finish.assert_called_with( - "topname", "topdesc", reporting.status.WARN) + "topname", "topdesc", events.status.WARN) - @mock.patch('cloudinit.reporting.report_finish_event') + @mock.patch('cloudinit.reporting.events.report_finish_event') def test_message_used_in_finish(self, report_finish): - with reporting.ReportEventStack("myname", "mydesc", - message="mymessage"): + with events.ReportEventStack("myname", "mydesc", + message="mymessage"): pass self.assertEqual( - [mock.call("myname", "mymessage", reporting.status.SUCCESS)], + [mock.call("myname", "mymessage", events.status.SUCCESS)], report_finish.call_args_list) - @mock.patch('cloudinit.reporting.report_finish_event') + @mock.patch('cloudinit.reporting.events.report_finish_event') def test_message_updatable(self, report_finish): - with reporting.ReportEventStack("myname", "mydesc") as c: + with events.ReportEventStack("myname", "mydesc") as c: c.message = "all good" self.assertEqual( - [mock.call("myname", "all good", reporting.status.SUCCESS)], + [mock.call("myname", "all good", events.status.SUCCESS)], report_finish.call_args_list) - @mock.patch('cloudinit.reporting.report_start_event') - @mock.patch('cloudinit.reporting.report_finish_event') + @mock.patch('cloudinit.reporting.events.report_start_event') + @mock.patch('cloudinit.reporting.events.report_finish_event') def test_reporting_disabled_does_not_report_events( self, report_start, report_finish): - with reporting.ReportEventStack("a", "b", reporting_enabled=False): + with events.ReportEventStack("a", "b", reporting_enabled=False): pass self.assertEqual(report_start.call_count, 0) self.assertEqual(report_finish.call_count, 0) - @mock.patch('cloudinit.reporting.report_start_event') - @mock.patch('cloudinit.reporting.report_finish_event') + @mock.patch('cloudinit.reporting.events.report_start_event') + @mock.patch('cloudinit.reporting.events.report_finish_event') def test_reporting_child_default_to_parent( self, report_start, report_finish): - parent = reporting.ReportEventStack( + parent = events.ReportEventStack( "pname", "pdesc", reporting_enabled=False) - child = reporting.ReportEventStack("cname", "cdesc", parent=parent) + child = events.ReportEventStack("cname", "cdesc", parent=parent) with parent: with child: pass @@ -343,17 +344,17 @@ class TestReportingEventStack(TestCase): self.assertEqual(report_finish.call_count, 0) def test_reporting_event_has_sane_repr(self): - myrep = reporting.ReportEventStack("fooname", "foodesc", - reporting_enabled=True).__repr__() + myrep = events.ReportEventStack("fooname", "foodesc", + reporting_enabled=True).__repr__() self.assertIn("fooname", myrep) self.assertIn("foodesc", myrep) self.assertIn("True", myrep) def test_set_invalid_result_raises_value_error(self): - f = reporting.ReportEventStack("myname", "mydesc") + f = events.ReportEventStack("myname", "mydesc") self.assertRaises(ValueError, setattr, f, "result", "BOGUS") class TestStatusAccess(TestCase): def test_invalid_status_access_raises_value_error(self): - self.assertRaises(AttributeError, getattr, reporting.status, "BOGUS") + self.assertRaises(AttributeError, getattr, events.status, "BOGUS") -- cgit v1.2.3 From 8cb7c3f7b5339e686bfbf95996b51afafeaf9c9e Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 3 Mar 2016 16:20:10 -0600 Subject: Update pep8 runner and fix pep8 issues --- Makefile | 9 ++-- bin/cloud-init | 43 +++++++++--------- cloudinit/config/cc_apt_configure.py | 6 ++- cloudinit/config/cc_disk_setup.py | 31 +++++++------ cloudinit/config/cc_grub_dpkg.py | 8 ++-- cloudinit/config/cc_keys_to_console.py | 2 +- cloudinit/config/cc_lxd.py | 2 +- cloudinit/config/cc_mounts.py | 12 ++--- cloudinit/config/cc_power_state_change.py | 2 +- cloudinit/config/cc_puppet.py | 6 +-- cloudinit/config/cc_resizefs.py | 2 +- cloudinit/config/cc_rh_subscription.py | 4 +- cloudinit/config/cc_set_hostname.py | 2 +- cloudinit/config/cc_ssh.py | 7 +-- cloudinit/config/cc_update_etc_hosts.py | 6 +-- cloudinit/config/cc_update_hostname.py | 2 +- cloudinit/config/cc_yum_add_repo.py | 2 +- cloudinit/distros/__init__.py | 12 ++--- cloudinit/distros/arch.py | 6 +-- cloudinit/distros/debian.py | 5 ++- cloudinit/distros/freebsd.py | 4 +- cloudinit/distros/gentoo.py | 4 +- cloudinit/distros/parsers/hostname.py | 2 +- cloudinit/distros/parsers/resolv_conf.py | 2 +- cloudinit/distros/parsers/sys_conf.py | 7 ++- cloudinit/filters/launch_index.py | 2 +- cloudinit/helpers.py | 7 +-- cloudinit/sources/DataSourceAzure.py | 21 +++++---- cloudinit/sources/DataSourceConfigDrive.py | 2 +- cloudinit/sources/DataSourceEc2.py | 10 ++--- cloudinit/sources/DataSourceMAAS.py | 15 ++++--- cloudinit/sources/DataSourceOVF.py | 4 +- cloudinit/sources/DataSourceOpenNebula.py | 3 +- cloudinit/sources/DataSourceSmartOS.py | 7 ++- cloudinit/ssh_util.py | 3 +- cloudinit/stages.py | 18 ++++---- cloudinit/url_helper.py | 6 +-- cloudinit/util.py | 15 ++++--- tests/unittests/test_data.py | 5 ++- tests/unittests/test_datasource/test_altcloud.py | 23 +++++----- tests/unittests/test_datasource/test_azure.py | 15 ++++--- .../unittests/test_datasource/test_configdrive.py | 12 ++--- tests/unittests/test_datasource/test_maas.py | 16 +++---- tests/unittests/test_datasource/test_smartos.py | 6 +-- .../test_handler/test_handler_power_state.py | 3 +- .../test_handler/test_handler_seed_random.py | 3 +- .../unittests/test_handler/test_handler_snappy.py | 3 +- tests/unittests/test_sshutil.py | 3 +- tests/unittests/test_templating.py | 3 +- tools/hacking.py | 16 +++---- tools/mock-meta.py | 27 +++++++----- tools/run-pep8 | 51 ++++++++-------------- 52 files changed, 244 insertions(+), 243 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/Makefile b/Makefile index 058ac199..fb65b70b 100644 --- a/Makefile +++ b/Makefile @@ -20,13 +20,14 @@ all: test check_version check: pep8 pyflakes pyflakes3 unittest pep8: - @$(CWD)/tools/run-pep8 $(PY_FILES) + @$(CWD)/tools/run-pep8 pyflakes: - @$(CWD)/tools/tox-venv py27 pyflakes $(PY_FILES) + @$(CWD)/tools/run-pyflakes -pyflakes: - @$(CWD)/tools/tox-venv py34 pyflakes $(PY_FILES) +pyflakes3: + @$(CWD)/tools/run-pyflakes3 + unittest: nosetests $(noseopts) tests/unittests diff --git a/bin/cloud-init b/bin/cloud-init index 9b90c45e..7f665e7e 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -194,7 +194,7 @@ def main_init(name, args): if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) + " longer be active shortly")) logging.resetLogging() logging.setupLogging(init.cfg) apply_reporting_cfg(init.cfg) @@ -276,9 +276,9 @@ def main_init(name, args): # This may run user-data handlers and/or perform # url downloads and such as needed. (ran, _results) = init.cloudify().run('consume_data', - init.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE) + init.consume_data, + args=[PER_INSTANCE], + freq=PER_INSTANCE) if not ran: # Just consume anything that is set to run per-always # if nothing ran in the per-instance code @@ -349,7 +349,7 @@ def main_modules(action_name, args): if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) + " longer be active shortly")) logging.resetLogging() logging.setupLogging(mods.cfg) apply_reporting_cfg(init.cfg) @@ -534,7 +534,8 @@ def status_wrapper(name, args, data_d=None, link_d=None): errors.extend(v1[m].get('errors', [])) atomic_write_json(result_path, - {'v1': {'datasource': v1['datasource'], 'errors': errors}}) + {'v1': {'datasource': v1['datasource'], + 'errors': errors}}) util.sym_link(os.path.relpath(result_path, link_d), result_link, force=True) @@ -578,13 +579,13 @@ def main(): # These settings are used for the 'config' and 'final' stages parser_mod = subparsers.add_parser('modules', - help=('activates modules ' - 'using a given configuration key')) + help=('activates modules using ' + 'a given configuration key')) parser_mod.add_argument("--mode", '-m', action='store', - help=("module configuration name " - "to use (default: %(default)s)"), - default='config', - choices=('init', 'config', 'final')) + help=("module configuration name " + "to use (default: %(default)s)"), + default='config', + choices=('init', 'config', 'final')) parser_mod.set_defaults(action=('modules', main_modules)) # These settings are used when you want to query information @@ -600,22 +601,22 @@ def main(): # This subcommand allows you to run a single module parser_single = subparsers.add_parser('single', - help=('run a single module ')) + help=('run a single module ')) parser_single.set_defaults(action=('single', main_single)) parser_single.add_argument("--name", '-n', action="store", - help="module name to run", - required=True) + help="module name to run", + required=True) parser_single.add_argument("--frequency", action="store", - help=("frequency of the module"), - required=False, - choices=list(FREQ_SHORT_NAMES.keys())) + help=("frequency of the module"), + required=False, + choices=list(FREQ_SHORT_NAMES.keys())) parser_single.add_argument("--report", action="store_true", help="enable reporting", required=False) parser_single.add_argument("module_args", nargs="*", - metavar='argument', - help=('any additional arguments to' - ' pass to this module')) + metavar='argument', + help=('any additional arguments to' + ' pass to this module')) parser_single.set_defaults(action=('single', main_single)) args = parser.parse_args() diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 9e9e9e26..702977cb 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -91,7 +91,8 @@ def handle(name, cfg, cloud, log, _args): if matchcfg: matcher = re.compile(matchcfg).search else: - matcher = lambda f: False + def matcher(x): + return False errors = add_sources(cfg['apt_sources'], params, aa_repo_match=matcher) @@ -173,7 +174,8 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): template_params = {} if aa_repo_match is None: - aa_repo_match = lambda f: False + def aa_repo_match(x): + return False errorlist = [] for ent in srclist: diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index d5b0d1d7..0ecc2e4c 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -167,11 +167,12 @@ def enumerate_disk(device, nodeps=False): parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0] for part in parts: - d = {'name': None, - 'type': None, - 'fstype': None, - 'label': None, - } + d = { + 'name': None, + 'type': None, + 'fstype': None, + 'label': None, + } for key, value in value_splitter(part): d[key.lower()] = value @@ -701,11 +702,12 @@ def lookup_force_flag(fs): """ A force flag might be -F or -F, this look it up """ - flags = {'ext': '-F', - 'btrfs': '-f', - 'xfs': '-f', - 'reiserfs': '-f', - } + flags = { + 'ext': '-F', + 'btrfs': '-f', + 'xfs': '-f', + 'reiserfs': '-f', + } if 'ext' in fs.lower(): fs = 'ext' @@ -824,10 +826,11 @@ def mkfs(fs_cfg): # Create the commands if fs_cmd: - fs_cmd = fs_cfg['cmd'] % {'label': label, - 'filesystem': fs_type, - 'device': device, - } + fs_cmd = fs_cfg['cmd'] % { + 'label': label, + 'filesystem': fs_type, + 'device': device, + } else: # Find the mkfs command mkfs_cmd = util.which("mkfs.%s" % fs_type) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 456597af..acd3e60a 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -38,11 +38,11 @@ def handle(name, cfg, _cloud, log, _args): idevs = util.get_cfg_option_str(mycfg, "grub-pc/install_devices", None) idevs_empty = util.get_cfg_option_str(mycfg, - "grub-pc/install_devices_empty", None) + "grub-pc/install_devices_empty", + None) if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") - and not os.path.exists("/dev/xvda"))): + (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): if idevs is None: idevs = "" if idevs_empty is None: @@ -66,7 +66,7 @@ def handle(name, cfg, _cloud, log, _args): (idevs, idevs_empty)) log.debug("Setting grub debconf-set-selections with '%s','%s'" % - (idevs, idevs_empty)) + (idevs, idevs_empty)) try: util.subp(['debconf-set-selections'], dconf_sel) diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index f1c1adff..aa844ee9 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -48,7 +48,7 @@ def handle(name, cfg, cloud, log, _args): "ssh_fp_console_blacklist", []) key_blacklist = util.get_cfg_option_list(cfg, "ssh_key_console_blacklist", - ["ssh-dss"]) + ["ssh-dss"]) try: cmd = [helper_path] diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 7d8a0202..e2fdf68e 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -59,7 +59,7 @@ def handle(name, cfg, cloud, log, args): if init_cfg: if not isinstance(init_cfg, dict): log.warn("lxd/init config must be a dictionary. found a '%s'", - type(init_cfg)) + type(init_cfg)) return cmd = ['lxd', 'init', '--auto'] for k in init_keys: diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 11089d8d..4fe3ee21 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -204,12 +204,12 @@ def setup_swapfile(fname, size=None, maxsize=None): try: util.ensure_dir(tdir) util.log_time(LOG.debug, msg, func=util.subp, - args=[['sh', '-c', - ('rm -f "$1" && umask 0066 && ' - '{ fallocate -l "${2}M" "$1" || ' - ' dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && ' - 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), - 'setup_swap', fname, mbsize]]) + args=[['sh', '-c', + ('rm -f "$1" && umask 0066 && ' + '{ fallocate -l "${2}M" "$1" || ' + ' dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && ' + 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), + 'setup_swap', fname, mbsize]]) except Exception as e: raise IOError("Failed %s: %s" % (msg, e)) diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 7d9567e3..cc3f7f70 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -105,7 +105,7 @@ def handle(_name, cfg, _cloud, log, _args): log.debug("After pid %s ends, will execute: %s" % (mypid, ' '.join(args))) - util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, + util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, condition, execmd, [args, devnull_fp]) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 4501598e..774d3322 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -36,8 +36,8 @@ def _autostart_puppet(log): # Set puppet to automatically start if os.path.exists('/etc/default/puppet'): util.subp(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet'], capture=False) + '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet'], capture=False) elif os.path.exists('/bin/systemctl'): util.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False) @@ -65,7 +65,7 @@ def handle(name, cfg, cloud, log, _args): " doing nothing.")) elif install: log.debug(("Attempting to install puppet %s,"), - version if version else 'latest') + version if version else 'latest') cloud.distro.install_packages(('puppet', version)) # ... and then update the puppet configuration diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index cbc07853..2a2a9f59 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -166,7 +166,7 @@ def handle(name, cfg, _cloud, log, args): func=do_resize, args=(resize_cmd, log)) else: util.log_time(logfunc=log.debug, msg="Resizing", - func=do_resize, args=(resize_cmd, log)) + func=do_resize, args=(resize_cmd, log)) action = 'Resized' if resize_root == NOBLOCK: diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 3b30c47e..6f474aed 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -127,8 +127,8 @@ class SubscriptionManager(object): return False, not_bool if (self.servicelevel is not None) and \ - ((not self.auto_attach) - or (util.is_false(str(self.auto_attach)))): + ((not self.auto_attach) or + (util.is_false(str(self.auto_attach)))): no_auto = ("The service-level key must be used in conjunction " "with the auto-attach key. Please re-run with " diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index 5d7f4331..f43d8d5a 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -24,7 +24,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not setting the hostname in module %s"), name) + " not setting the hostname in module %s"), name) return (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 5bd2dec6..d24e43c0 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -30,9 +30,10 @@ from cloudinit import distros as ds from cloudinit import ssh_util from cloudinit import util -DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding," -"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " -"rather than the user \\\"root\\\".\';echo;sleep 10\"") +DISABLE_ROOT_OPTS = ( + "no-port-forwarding,no-agent-forwarding," + "no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\"" + " rather than the user \\\"root\\\".\';echo;sleep 10\"") GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa', 'ed25519'] KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index d3dd1f32..15703efe 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -41,10 +41,10 @@ def handle(name, cfg, cloud, log, _args): if not tpl_fn_name: raise RuntimeError(("No hosts template could be" " found for distro %s") % - (cloud.distro.osfamily)) + (cloud.distro.osfamily)) templater.render_to_file(tpl_fn_name, '/etc/hosts', - {'hostname': hostname, 'fqdn': fqdn}) + {'hostname': hostname, 'fqdn': fqdn}) elif manage_hosts == "localhost": (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) @@ -57,4 +57,4 @@ def handle(name, cfg, cloud, log, _args): cloud.distro.update_etc_hosts(hostname, fqdn) else: log.debug(("Configuration option 'manage_etc_hosts' is not set," - " not managing /etc/hosts in module %s"), name) + " not managing /etc/hosts in module %s"), name) diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index e396ba13..5b78afe1 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -29,7 +29,7 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not updating the hostname in module %s"), name) + " not updating the hostname in module %s"), name) return (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 3b821af9..64fba869 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -92,7 +92,7 @@ def handle(name, cfg, _cloud, log, _args): for req_field in ['baseurl']: if req_field not in repo_config: log.warn(("Repository %s does not contain a %s" - " configuration 'required' entry"), + " configuration 'required' entry"), repo_id, req_field) missing_required += 1 if not missing_required: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 71884b32..661a9fd2 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -211,8 +211,8 @@ class Distro(object): # If the system hostname is different than the previous # one or the desired one lets update it as well - if (not sys_hostname) or (sys_hostname == prev_hostname - and sys_hostname != hostname): + if ((not sys_hostname) or (sys_hostname == prev_hostname and + sys_hostname != hostname)): update_files.append(sys_fn) # If something else has changed the hostname after we set it @@ -221,7 +221,7 @@ class Distro(object): if (sys_hostname and prev_hostname and sys_hostname != prev_hostname): LOG.info("%s differs from %s, assuming user maintained hostname.", - prev_hostname_fn, sys_fn) + prev_hostname_fn, sys_fn) return # Remove duplicates (incase the previous config filename) @@ -289,7 +289,7 @@ class Distro(object): def _bring_up_interface(self, device_name): cmd = ['ifup', device_name] LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) + device_name, cmd) try: (_out, err) = util.subp(cmd) if len(err): @@ -548,7 +548,7 @@ class Distro(object): for member in members: if not util.is_user(member): LOG.warn("Unable to add group member '%s' to group '%s'" - "; user does not exist.", member, name) + "; user does not exist.", member, name) continue util.subp(['usermod', '-a', '-G', name, member]) @@ -886,7 +886,7 @@ def fetch(name): locs, looked_locs = importer.find_module(name, ['', __name__], ['Distro']) if not locs: raise ImportError("No distribution found for distro %s (searched %s)" - % (name, looked_locs)) + % (name, looked_locs)) mod = importer.import_module(locs[0]) cls = getattr(mod, 'Distro') return cls diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 45fcf26f..93a2e008 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -74,7 +74,7 @@ class Distro(distros.Distro): 'Interface': dev, 'IP': info.get('bootproto'), 'Address': "('%s/%s')" % (info.get('address'), - info.get('netmask')), + info.get('netmask')), 'Gateway': info.get('gateway'), 'DNS': str(tuple(info.get('dns-nameservers'))).replace(',', '') } @@ -86,7 +86,7 @@ class Distro(distros.Distro): if nameservers: util.write_file(self.resolve_conf_fn, - convert_resolv_conf(nameservers)) + convert_resolv_conf(nameservers)) return dev_names @@ -102,7 +102,7 @@ class Distro(distros.Distro): def _bring_up_interface(self, device_name): cmd = ['netctl', 'restart', device_name] LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) + device_name, cmd) try: (_out, err) = util.subp(cmd) if len(err): diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 6d3a82bf..db5890b1 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -159,8 +159,9 @@ class Distro(distros.Distro): # Allow the output of this to flow outwards (ie not be captured) util.log_time(logfunc=LOG.debug, - msg="apt-%s [%s]" % (command, ' '.join(cmd)), func=util.subp, - args=(cmd,), kwargs={'env': e, 'capture': False}) + msg="apt-%s [%s]" % (command, ' '.join(cmd)), + func=util.subp, + args=(cmd,), kwargs={'env': e, 'capture': False}) def update_package_sources(self): self._runner.run("update-sources", self.package_command, diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index 4c484639..72012056 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -205,8 +205,8 @@ class Distro(distros.Distro): redact_opts = ['passwd'] for key, val in kwargs.items(): - if (key in adduser_opts and val - and isinstance(val, six.string_types)): + if (key in adduser_opts and val and + isinstance(val, six.string_types)): adduser_cmd.extend([adduser_opts[key], val]) # Redact certain fields from the logs diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 9e80583c..6267dd6e 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -66,7 +66,7 @@ class Distro(distros.Distro): def _bring_up_interface(self, device_name): cmd = ['/etc/init.d/net.%s' % device_name, 'restart'] LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) + device_name, cmd) try: (_out, err) = util.subp(cmd) if len(err): @@ -88,7 +88,7 @@ class Distro(distros.Distro): (_out, err) = util.subp(cmd) if len(err): LOG.warn("Running %s resulted in stderr output: %s", cmd, - err) + err) except util.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False diff --git a/cloudinit/distros/parsers/hostname.py b/cloudinit/distros/parsers/hostname.py index 84a1de42..efb185d4 100644 --- a/cloudinit/distros/parsers/hostname.py +++ b/cloudinit/distros/parsers/hostname.py @@ -84,5 +84,5 @@ class HostnameConf(object): hostnames_found.add(head) if len(hostnames_found) > 1: raise IOError("Multiple hostnames (%s) found!" - % (hostnames_found)) + % (hostnames_found)) return entries diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py index 8aee03a4..2ed13d9c 100644 --- a/cloudinit/distros/parsers/resolv_conf.py +++ b/cloudinit/distros/parsers/resolv_conf.py @@ -132,7 +132,7 @@ class ResolvConf(object): # Some hard limit on 256 chars total raise ValueError(("Adding %r would go beyond the " "256 maximum search list character limit") - % (search_domain)) + % (search_domain)) self._remove_option('search') self._contents.append(('option', ['search', s_list, ''])) return flat_sds diff --git a/cloudinit/distros/parsers/sys_conf.py b/cloudinit/distros/parsers/sys_conf.py index d795e12f..6157cf32 100644 --- a/cloudinit/distros/parsers/sys_conf.py +++ b/cloudinit/distros/parsers/sys_conf.py @@ -77,8 +77,7 @@ class SysConf(configobj.ConfigObj): quot_func = None if value[0] in ['"', "'"] and value[-1] in ['"', "'"]: if len(value) == 1: - quot_func = (lambda x: - self._get_single_quote(x) % x) + quot_func = (lambda x: self._get_single_quote(x) % x) else: # Quote whitespace if it isn't the start + end of a shell command if value.strip().startswith("$(") and value.strip().endswith(")"): @@ -91,10 +90,10 @@ class SysConf(configobj.ConfigObj): # to use single quotes which won't get expanded... if re.search(r"[\n\"']", value): quot_func = (lambda x: - self._get_triple_quote(x) % x) + self._get_triple_quote(x) % x) else: quot_func = (lambda x: - self._get_single_quote(x) % x) + self._get_single_quote(x) % x) else: quot_func = pipes.quote if not quot_func: diff --git a/cloudinit/filters/launch_index.py b/cloudinit/filters/launch_index.py index 5bebd318..baecdac9 100644 --- a/cloudinit/filters/launch_index.py +++ b/cloudinit/filters/launch_index.py @@ -61,7 +61,7 @@ class Filter(object): discarded += 1 LOG.debug(("Discarding %s multipart messages " "which do not match launch index %s"), - discarded, self.wanted_idx) + discarded, self.wanted_idx) new_message = copy.copy(message) new_message.set_payload(new_msgs) new_message[ud.ATTACHMENT_FIELD] = str(len(new_msgs)) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 5e99d185..a6eb20fe 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -139,9 +139,10 @@ class FileSemaphores(object): # but the item had run before we did canon_sem_name. if cname != name and os.path.exists(self._get_path(name, freq)): LOG.warn("%s has run without canonicalized name [%s].\n" - "likely the migrator has not yet run. It will run next boot.\n" - "run manually with: cloud-init single --name=migrator" - % (name, cname)) + "likely the migrator has not yet run. " + "It will run next boot.\n" + "run manually with: cloud-init single --name=migrator" + % (name, cname)) return True return False diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index bd80a8a6..b03ab895 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -38,7 +38,8 @@ LOG = logging.getLogger(__name__) DS_NAME = 'Azure' DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] -BOUNCE_COMMAND = ['sh', '-xc', +BOUNCE_COMMAND = [ + 'sh', '-xc', "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"] BUILTIN_DS_CONFIG = { @@ -91,9 +92,9 @@ def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'): """ policy = cfg['hostname_bounce']['policy'] previous_hostname = get_hostname(hostname_command) - if (not util.is_true(cfg.get('set_hostname')) - or util.is_false(policy) - or (previous_hostname == temp_hostname and policy != 'force')): + if (not util.is_true(cfg.get('set_hostname')) or + util.is_false(policy) or + (previous_hostname == temp_hostname and policy != 'force')): yield None return set_hostname(temp_hostname, hostname_command) @@ -123,8 +124,8 @@ class DataSourceAzureNet(sources.DataSource): with temporary_hostname(temp_hostname, self.ds_cfg, hostname_command=hostname_command) \ as previous_hostname: - if (previous_hostname is not None - and util.is_true(self.ds_cfg.get('set_hostname'))): + if (previous_hostname is not None and + util.is_true(self.ds_cfg.get('set_hostname'))): cfg = self.ds_cfg['hostname_bounce'] try: perform_hostname_bounce(hostname=temp_hostname, @@ -152,7 +153,8 @@ class DataSourceAzureNet(sources.DataSource): else: bname = str(pk['fingerprint'] + ".crt") fp_files += [os.path.join(ddir, bname)] - LOG.debug("ssh authentication: using fingerprint from fabirc") + LOG.debug("ssh authentication: " + "using fingerprint from fabirc") missing = util.log_time(logfunc=LOG.debug, msg="waiting for files", func=wait_for_files, @@ -506,7 +508,7 @@ def read_azure_ovf(contents): raise BrokenAzureDataSource("invalid xml: %s" % e) results = find_child(dom.documentElement, - lambda n: n.localName == "ProvisioningSection") + lambda n: n.localName == "ProvisioningSection") if len(results) == 0: raise NonAzureDataSource("No ProvisioningSection") @@ -516,7 +518,8 @@ def read_azure_ovf(contents): provSection = results[0] lpcs_nodes = find_child(provSection, - lambda n: n.localName == "LinuxProvisioningConfigurationSet") + lambda n: + n.localName == "LinuxProvisioningConfigurationSet") if len(results) == 0: raise NonAzureDataSource("No LinuxProvisioningConfigurationSet") diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index eb474079..e3916208 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -39,7 +39,7 @@ FS_TYPES = ('vfat', 'iso9660') LABEL_TYPES = ('config-2',) POSSIBLE_MOUNTS = ('sr', 'cd') OPTICAL_DEVICES = tuple(('/dev/%s%s' % (z, i) for z in POSSIBLE_MOUNTS - for i in range(0, 2))) + for i in range(0, 2))) class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 0032d06c..6a897f7d 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -61,12 +61,12 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = ec2.get_instance_userdata(self.api_ver, - self.metadata_address) + self.userdata_raw = \ + ec2.get_instance_userdata(self.api_ver, self.metadata_address) self.metadata = ec2.get_instance_metadata(self.api_ver, self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", - int(time.time() - start_time)) + int(time.time() - start_time)) return True except Exception: util.logexc(LOG, "Failed reading from metadata address %s", @@ -132,13 +132,13 @@ class DataSourceEc2(sources.DataSource): start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn) + timeout=timeout, status_cb=LOG.warn) if url: LOG.debug("Using metadata source: '%s'", url2base[url]) else: LOG.critical("Giving up on md from %s after %s seconds", - urls, int(time.time() - start_time)) + urls, int(time.time() - start_time)) self.metadata_address = url2base.get(url) return bool(url) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index cfc59ca5..f18c4cee 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -275,17 +275,18 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description='Interact with MAAS DS') parser.add_argument("--config", metavar="file", - help="specify DS config file", default=None) + help="specify DS config file", default=None) parser.add_argument("--ckey", metavar="key", - help="the consumer key to auth with", default=None) + help="the consumer key to auth with", default=None) parser.add_argument("--tkey", metavar="key", - help="the token key to auth with", default=None) + help="the token key to auth with", default=None) parser.add_argument("--csec", metavar="secret", - help="the consumer secret (likely '')", default="") + help="the consumer secret (likely '')", default="") parser.add_argument("--tsec", metavar="secret", - help="the token secret to auth with", default=None) + help="the token secret to auth with", default=None) parser.add_argument("--apiver", metavar="version", - help="the apiver to use ("" can be used)", default=MD_VERSION) + help="the apiver to use ("" can be used)", + default=MD_VERSION) subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") subcmds.add_parser('crawl', help="crawl the datasource") @@ -297,7 +298,7 @@ if __name__ == "__main__": args = parser.parse_args() creds = {'consumer_key': args.ckey, 'token_key': args.tkey, - 'token_secret': args.tsec, 'consumer_secret': args.csec} + 'token_secret': args.tsec, 'consumer_secret': args.csec} if args.config: cfg = util.read_conf(args.config) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 58a4b2a2..adf9b12e 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -264,14 +264,14 @@ def get_properties(contents): # could also check here that elem.namespaceURI == # "http://schemas.dmtf.org/ovf/environment/1" propSections = find_child(dom.documentElement, - lambda n: n.localName == "PropertySection") + lambda n: n.localName == "PropertySection") if len(propSections) == 0: raise XmlError("No 'PropertySection's") props = {} propElems = find_child(propSections[0], - (lambda n: n.localName == "Property")) + (lambda n: n.localName == "Property")) for elem in propElems: key = elem.attributes.getNamedItemNS(envNsURI, "key").value diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index ac2c3b45..b26940d1 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -404,7 +404,8 @@ def read_context_disk_dir(source_dir, asuser=None): if ssh_key_var: lines = context.get(ssh_key_var).splitlines() results['metadata']['public-keys'] = [l for l in lines - if len(l) and not l.startswith("#")] + if len(l) and not + l.startswith("#")] # custom hostname -- try hostname or leave cloud-init # itself create hostname from IP address later diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 7453379a..139ee52c 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -90,8 +90,7 @@ BUILTIN_DS_CONFIG = { 'user-data', 'user-script', 'sdc:datacenter_name', - 'sdc:uuid', - ], + 'sdc:uuid'], 'base64_keys': [], 'base64_all': False, 'disk_aliases': {'ephemeral0': '/dev/vdb'}, @@ -450,7 +449,7 @@ class JoyentMetadataClient(object): response = bytearray() response.extend(self.metasource.read(1)) - while response[-1:] != b'\n': + while response[-1:] != b'\n': response.extend(self.metasource.read(1)) response = response.rstrip().decode('ascii') LOG.debug('Read "%s" from metadata transport.', response) @@ -513,7 +512,7 @@ def write_boot_content(content, content_f, link=None, shebang=False, except Exception as e: util.logexc(LOG, ("Failed to identify script type for %s" % - content_f, e)) + content_f, e)) if link: try: diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 9b2f5ed5..c74a7ae2 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -31,7 +31,8 @@ LOG = logging.getLogger(__name__) DEF_SSHD_CFG = "/etc/ssh/sshd_config" # taken from openssh source key.c/key_type_from_name -VALID_KEY_TYPES = ("rsa", "dsa", "ssh-rsa", "ssh-dss", "ecdsa", +VALID_KEY_TYPES = ( + "rsa", "dsa", "ssh-rsa", "ssh-dss", "ecdsa", "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com", "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com", "ssh-rsa-cert-v01@openssh.com", "ssh-dss-cert-v01@openssh.com", diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 9f192c8d..dbcf3d55 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -509,13 +509,13 @@ class Init(object): def consume_data(self, frequency=PER_INSTANCE): # Consume the userdata first, because we need want to let the part # handlers run first (for merging stuff) - with events.ReportEventStack( - "consume-user-data", "reading and applying user-data", - parent=self.reporter): + with events.ReportEventStack("consume-user-data", + "reading and applying user-data", + parent=self.reporter): self._consume_userdata(frequency) - with events.ReportEventStack( - "consume-vendor-data", "reading and applying vendor-data", - parent=self.reporter): + with events.ReportEventStack("consume-vendor-data", + "reading and applying vendor-data", + parent=self.reporter): self._consume_vendordata(frequency) # Perform post-consumption adjustments so that @@ -655,7 +655,7 @@ class Modules(object): else: raise TypeError(("Failed to read '%s' item in config," " unknown type %s") % - (item, type_utils.obj_name(item))) + (item, type_utils.obj_name(item))) return module_list def _fixup_modules(self, raw_mods): @@ -762,8 +762,8 @@ class Modules(object): if skipped: LOG.info("Skipping modules %s because they are not verified " - "on distro '%s'. To run anyway, add them to " - "'unverified_modules' in config.", skipped, d_name) + "on distro '%s'. To run anyway, add them to " + "'unverified_modules' in config.", skipped, d_name) if forced: LOG.info("running unverified_modules: %s", forced) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index f2e1390e..936f7da5 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -252,9 +252,9 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # attrs return UrlResponse(r) except exceptions.RequestException as e: - if (isinstance(e, (exceptions.HTTPError)) - and hasattr(e, 'response') # This appeared in v 0.10.8 - and hasattr(e.response, 'status_code')): + if (isinstance(e, (exceptions.HTTPError)) and + hasattr(e, 'response') and # This appeared in v 0.10.8 + hasattr(e.response, 'status_code')): excps.append(UrlError(e, code=e.response.status_code, headers=e.response.headers, url=url)) diff --git a/cloudinit/util.py b/cloudinit/util.py index 45d49e66..de37b0f5 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -612,7 +612,7 @@ def redirect_output(outfmt, errfmt, o_out=None, o_err=None): def make_url(scheme, host, port=None, - path='', params='', query='', fragment=''): + path='', params='', query='', fragment=''): pieces = [] pieces.append(scheme or '') @@ -804,8 +804,8 @@ def load_yaml(blob, default=None, allowed=(dict,)): blob = decode_binary(blob) try: LOG.debug("Attempting to load yaml from string " - "of length %s with allowed root types %s", - len(blob), allowed) + "of length %s with allowed root types %s", + len(blob), allowed) converted = safeyaml.load(blob) if not isinstance(converted, allowed): # Yes this will just be caught, but thats ok for now... @@ -878,7 +878,7 @@ def read_conf_with_confd(cfgfile): if not isinstance(confd, six.string_types): raise TypeError(("Config file %s contains 'conf_d' " "with non-string type %s") % - (cfgfile, type_utils.obj_name(confd))) + (cfgfile, type_utils.obj_name(confd))) else: confd = str(confd).strip() elif os.path.isdir("%s.d" % cfgfile): @@ -1041,7 +1041,8 @@ def is_resolvable(name): for iname in badnames: try: result = socket.getaddrinfo(iname, None, 0, 0, - socket.SOCK_STREAM, socket.AI_CANONNAME) + socket.SOCK_STREAM, + socket.AI_CANONNAME) badresults[iname] = [] for (_fam, _stype, _proto, cname, sockaddr) in result: badresults[iname].append("%s: %s" % (cname, sockaddr[0])) @@ -1109,7 +1110,7 @@ def close_stdin(): def find_devs_with(criteria=None, oformat='device', - tag=None, no_cache=False, path=None): + tag=None, no_cache=False, path=None): """ find devices matching given criteria (via blkid) criteria can be *one* of: @@ -1628,7 +1629,7 @@ def write_file(filename, content, mode=0o644, omode="wb"): content = decode_binary(content) write_type = 'characters' LOG.debug("Writing to %s - %s: [%s] %s %s", - filename, omode, mode, len(content), write_type) + filename, omode, mode, len(content), write_type) with SeLinuxGuard(path=filename): with open(filename, omode) as fh: fh.write(content) diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index c603bfdb..9c1ec1d4 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -27,11 +27,12 @@ from cloudinit import stages from cloudinit import user_data as ud from cloudinit import util -INSTANCE_ID = "i-testing" - from . import helpers +INSTANCE_ID = "i-testing" + + class FakeDataSource(sources.DataSource): def __init__(self, userdata=None, vendordata=None): diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index e9cd2fa5..85759c68 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -134,8 +134,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('RHEV') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('RHEV', \ - dsrc.get_cloud_type()) + self.assertEquals('RHEV', dsrc.get_cloud_type()) def test_vsphere(self): ''' @@ -144,8 +143,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('VMware Virtual Platform') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('VSPHERE', \ - dsrc.get_cloud_type()) + self.assertEquals('VSPHERE', dsrc.get_cloud_type()) def test_unknown(self): ''' @@ -154,8 +152,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('Unrecognized Platform') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('UNKNOWN', \ - dsrc.get_cloud_type()) + self.assertEquals('UNKNOWN', dsrc.get_cloud_type()) class TestGetDataCloudInfoFile(TestCase): @@ -412,27 +409,27 @@ class TestReadUserDataCallback(TestCase): '''Test read_user_data_callback() with both files.''' self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + read_user_data_callback(self.mount_dir)) def test_callback_dc(self): '''Test read_user_data_callback() with only DC file.''' _remove_user_data_files(self.mount_dir, - dc_file=False, - non_dc_file=True) + dc_file=False, + non_dc_file=True) self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + read_user_data_callback(self.mount_dir)) def test_callback_non_dc(self): '''Test read_user_data_callback() with only non-DC file.''' _remove_user_data_files(self.mount_dir, - dc_file=True, - non_dc_file=False) + dc_file=True, + non_dc_file=False) self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + read_user_data_callback(self.mount_dir)) def test_callback_none(self): '''Test read_user_data_callback() no files are found.''' diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 3933794f..4c9c7d8b 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -207,7 +207,7 @@ class TestAzureDataSource(TestCase): yaml_cfg = "{agent_command: my_command}\n" cfg = yaml.safe_load(yaml_cfg) odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}} + 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) @@ -219,8 +219,8 @@ class TestAzureDataSource(TestCase): # set dscfg in via base64 encoded yaml cfg = {'agent_command': "my_command"} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': b64e(yaml.dump(cfg)), - 'encoding': 'base64'}} + 'dscfg': {'text': b64e(yaml.dump(cfg)), + 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) @@ -267,7 +267,8 @@ class TestAzureDataSource(TestCase): # should equal that after the '$' pos = defuser['passwd'].rfind("$") + 1 self.assertEqual(defuser['passwd'], - crypt.crypt(odata['UserPassword'], defuser['passwd'][0:pos])) + crypt.crypt(odata['UserPassword'], + defuser['passwd'][0:pos])) def test_userdata_plain(self): mydata = "FOOBAR" @@ -364,8 +365,8 @@ class TestAzureDataSource(TestCase): # Make sure that user can affect disk aliases dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': b64e(yaml.dump(dscfg)), - 'encoding': 'base64'}} + 'dscfg': {'text': b64e(yaml.dump(dscfg)), + 'encoding': 'base64'}} usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'}, 'ephemeral0': False}} userdata = '#cloud-config' + yaml.dump(usercfg) + "\n" @@ -634,7 +635,7 @@ class TestReadAzureOvf(TestCase): def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "" + construct_valid_ovf_env(data={}) self.assertRaises(DataSourceAzure.BrokenAzureDataSource, - DataSourceAzure.read_azure_ovf, invalid_xml) + DataSourceAzure.read_azure_ovf, invalid_xml) def test_load_with_pubkeys(self): mypklist = [{'fingerprint': 'fp1', 'path': 'path1', 'value': ''}] diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 83aca505..3954ceb3 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -293,9 +293,8 @@ class TestConfigDriveDataSource(TestCase): util.is_partition = my_is_partition devs_with_answers = {"TYPE=vfat": [], - "TYPE=iso9660": ["/dev/vdb"], - "LABEL=config-2": ["/dev/vdb"], - } + "TYPE=iso9660": ["/dev/vdb"], + "LABEL=config-2": ["/dev/vdb"]} self.assertEqual(["/dev/vdb"], ds.find_candidate_devs()) # add a vfat item @@ -306,9 +305,10 @@ class TestConfigDriveDataSource(TestCase): # verify that partitions are considered, that have correct label. devs_with_answers = {"TYPE=vfat": ["/dev/sda1"], - "TYPE=iso9660": [], "LABEL=config-2": ["/dev/vdb3"]} + "TYPE=iso9660": [], + "LABEL=config-2": ["/dev/vdb3"]} self.assertEqual(["/dev/vdb3"], - ds.find_candidate_devs()) + ds.find_candidate_devs()) finally: util.find_devs_with = orig_find_devs_with @@ -319,7 +319,7 @@ class TestConfigDriveDataSource(TestCase): populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) self.assertEqual(myds.get_public_ssh_keys(), - [OSTACK_META['public_keys']['mykey']]) + [OSTACK_META['public_keys']['mykey']]) def cfg_ds_from_dir(seed_d): diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index eb97b692..77d15cac 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -25,9 +25,9 @@ class TestMAASDataSource(TestCase): """Verify a valid seeddir is read as such.""" data = {'instance-id': 'i-valid01', - 'local-hostname': 'valid01-hostname', - 'user-data': b'valid01-userdata', - 'public-keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname'} + 'local-hostname': 'valid01-hostname', + 'user-data': b'valid01-userdata', + 'public-keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname'} my_d = os.path.join(self.tmp, "valid") populate_dir(my_d, data) @@ -45,8 +45,8 @@ class TestMAASDataSource(TestCase): """Verify extra files do not affect seed_dir validity.""" data = {'instance-id': 'i-valid-extra', - 'local-hostname': 'valid-extra-hostname', - 'user-data': b'valid-extra-userdata', 'foo': 'bar'} + 'local-hostname': 'valid-extra-hostname', + 'user-data': b'valid-extra-userdata', 'foo': 'bar'} my_d = os.path.join(self.tmp, "valid_extra") populate_dir(my_d, data) @@ -64,7 +64,7 @@ class TestMAASDataSource(TestCase): """Verify that invalid seed_dir raises MAASSeedDirMalformed.""" valid = {'instance-id': 'i-instanceid', - 'local-hostname': 'test-hostname', 'user-data': ''} + 'local-hostname': 'test-hostname', 'user-data': ''} my_based = os.path.join(self.tmp, "valid_extra") @@ -94,8 +94,8 @@ class TestMAASDataSource(TestCase): def test_seed_dir_missing(self): """Verify that missing seed_dir raises MAASSeedDirNone.""" self.assertRaises(DataSourceMAAS.MAASSeedDirNone, - DataSourceMAAS.read_maas_seed_dir, - os.path.join(self.tmp, "nonexistantdirectory")) + DataSourceMAAS.read_maas_seed_dir, + os.path.join(self.tmp, "nonexistantdirectory")) def test_seed_url_valid(self): """Verify that valid seed_url is read as such.""" diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 1235436d..5e617b83 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -463,8 +463,8 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): payloadstr = ' {0}'.format(self.response_parts['payload']) return ('V2 {length} {crc} {request_id} ' '{command}{payloadstr}\n'.format( - payloadstr=payloadstr, - **self.response_parts).encode('ascii')) + payloadstr=payloadstr, + **self.response_parts).encode('ascii')) self.metasource_data = None @@ -501,7 +501,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): written_line = self.serial.write.call_args[0][0] print(type(written_line)) self.assertEndsWith(written_line.decode('ascii'), - b'\n'.decode('ascii')) + b'\n'.decode('ascii')) self.assertEqual(1, written_line.count(b'\n')) def _get_written_line(self, key='some_key'): diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 5687b10d..f9660ff6 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -74,7 +74,7 @@ class TestLoadPowerState(t_help.TestCase): class TestCheckCondition(t_help.TestCase): def cmd_with_exit(self, rc): return([sys.executable, '-c', 'import sys; sys.exit(%s)' % rc]) - + def test_true_is_true(self): self.assertEqual(psc.check_condition(True), True) @@ -94,7 +94,6 @@ class TestCheckCondition(t_help.TestCase): self.assertEqual(mocklog.warn.call_count, 1) - def check_lps_ret(psc_return, mode=None): if len(psc_return) != 3: raise TypeError("length returned = %d" % len(psc_return)) diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index 0bcdcb31..34d11f21 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -190,7 +190,8 @@ class TestRandomSeed(t_help.TestCase): c = self._get_cloud('ubuntu', {}) self.whichdata = {} self.assertRaises(ValueError, cc_seed_random.handle, - 'test', {'random_seed': {'command_required': True}}, c, LOG, []) + 'test', {'random_seed': {'command_required': True}}, + c, LOG, []) def test_seed_command_and_required(self): c = self._get_cloud('ubuntu', {}) diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index eceb14d9..8aeff53c 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -125,8 +125,7 @@ class TestInstallPackages(t_help.TestCase): "pkg1.smoser.config": "pkg1.smoser.config-data", "pkg1.config": "pkg1.config-data", "pkg2.smoser_0.0_amd64.snap": "pkg2-snapdata", - "pkg2.smoser_0.0_amd64.config": "pkg2.config", - }) + "pkg2.smoser_0.0_amd64.config": "pkg2.config"}) ret = get_package_ops( packages=[], configs={}, installed=[], fspath=self.tmp) diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index 3b317121..9aeb1cde 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -32,7 +32,8 @@ VALID_CONTENT = { ), } -TEST_OPTIONS = ("no-port-forwarding,no-agent-forwarding,no-X11-forwarding," +TEST_OPTIONS = ( + "no-port-forwarding,no-agent-forwarding,no-X11-forwarding," 'command="echo \'Please login as the user \"ubuntu\" rather than the' 'user \"root\".\';echo;sleep 10"') diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 0c19a2c2..b9863650 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -114,5 +114,6 @@ $a,$b''' codename) out_data = templater.basic_render(in_data, - {'mirror': mirror, 'codename': codename}) + {'mirror': mirror, + 'codename': codename}) self.assertEqual(ex_data, out_data) diff --git a/tools/hacking.py b/tools/hacking.py index 3175df38..1a0631c2 100755 --- a/tools/hacking.py +++ b/tools/hacking.py @@ -47,10 +47,10 @@ def import_normalize(line): # handle "from x import y as z" to "import x.y as z" split_line = line.split() if (line.startswith("from ") and "," not in line and - split_line[2] == "import" and split_line[3] != "*" and - split_line[1] != "__future__" and - (len(split_line) == 4 or - (len(split_line) == 6 and split_line[4] == "as"))): + split_line[2] == "import" and split_line[3] != "*" and + split_line[1] != "__future__" and + (len(split_line) == 4 or + (len(split_line) == 6 and split_line[4] == "as"))): return "import %s.%s" % (split_line[1], split_line[3]) else: return line @@ -74,7 +74,7 @@ def cloud_import_alphabetical(physical_line, line_number, lines): split_line[0] == "import" and split_previous[0] == "import"): if split_line[1] < split_previous[1]: return (0, "N306: imports not in alphabetical order (%s, %s)" - % (split_previous[1], split_line[1])) + % (split_previous[1], split_line[1])) def cloud_docstring_start_space(physical_line): @@ -87,8 +87,8 @@ def cloud_docstring_start_space(physical_line): pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start if (pos != -1 and len(physical_line) > pos + 1): if (physical_line[pos + 3] == ' '): - return (pos, "N401: one line docstring should not start with" - " a space") + return (pos, + "N401: one line docstring should not start with a space") def cloud_todo_format(physical_line): @@ -167,4 +167,4 @@ if __name__ == "__main__": finally: if len(_missingImport) > 0: print >> sys.stderr, ("%i imports missing in this test environment" - % len(_missingImport)) + % len(_missingImport)) diff --git a/tools/mock-meta.py b/tools/mock-meta.py index dfbc2a71..1c746f17 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -126,11 +126,11 @@ class WebException(Exception): def yamlify(data): formatted = yaml.dump(data, - line_break="\n", - indent=4, - explicit_start=True, - explicit_end=True, - default_flow_style=False) + line_break="\n", + indent=4, + explicit_start=True, + explicit_end=True, + default_flow_style=False) return formatted @@ -282,7 +282,7 @@ class MetaDataHandler(object): else: log.warn(("Did not implement action %s, " "returning empty response: %r"), - action, NOT_IMPL_RESPONSE) + action, NOT_IMPL_RESPONSE) return NOT_IMPL_RESPONSE @@ -404,14 +404,17 @@ def setup_logging(log_level, fmt='%(levelname)s: @%(name)s : %(message)s'): def extract_opts(): parser = OptionParser() parser.add_option("-p", "--port", dest="port", action="store", type=int, - default=80, metavar="PORT", - help="port from which to serve traffic (default: %default)") + default=80, metavar="PORT", + help=("port from which to serve traffic" + " (default: %default)")) parser.add_option("-a", "--addr", dest="address", action="store", type=str, - default='0.0.0.0', metavar="ADDRESS", - help="address from which to serve traffic (default: %default)") + default='0.0.0.0', metavar="ADDRESS", + help=("address from which to serve traffic" + " (default: %default)")) parser.add_option("-f", '--user-data-file', dest='user_data_file', - action='store', metavar='FILE', - help="user data filename to serve back to incoming requests") + action='store', metavar='FILE', + help=("user data filename to serve back to" + "incoming requests")) (options, args) = parser.parse_args() out = dict() out['extra'] = args diff --git a/tools/run-pep8 b/tools/run-pep8 index ccd6be5a..086400fc 100755 --- a/tools/run-pep8 +++ b/tools/run-pep8 @@ -1,39 +1,22 @@ #!/bin/bash -if [ $# -eq 0 ]; then - files=( bin/cloud-init $(find * -name "*.py" -type f) ) +pycheck_dirs=( "cloudinit/" "bin/" "tests/" "tools/" ) +# FIXME: cloud-init modifies sys module path, pep8 does not like +# bin_files=( "bin/cloud-init" ) +CR=" +" +[ "$1" = "-v" ] && { verbose="$1"; shift; } || verbose="" + +set -f +if [ $# -eq 0 ]; then unset IFS + IFS="$CR" + files=( "${bin_files[@]}" "${pycheck_dirs[@]}" ) + unset IFS else - files=( "$@" ); + files=( "$@" ) fi -if [ -f 'hacking.py' ] -then - base=`pwd` -else - base=`pwd`/tools/ -fi - -IGNORE="" - -# King Arthur: Be quiet! ... Be Quiet! I Order You to Be Quiet. -IGNORE="$IGNORE,E121" # Continuation line indentation is not a multiple of four -IGNORE="$IGNORE,E123" # Closing bracket does not match indentation of opening bracket's line -IGNORE="$IGNORE,E124" # Closing bracket missing visual indentation -IGNORE="$IGNORE,E125" # Continuation line does not distinguish itself from next logical line -IGNORE="$IGNORE,E126" # Continuation line over-indented for hanging indent -IGNORE="$IGNORE,E127" # Continuation line over-indented for visual indent -IGNORE="$IGNORE,E128" # Continuation line under-indented for visual indent -IGNORE="$IGNORE,E502" # The backslash is redundant between brackets -IGNORE="${IGNORE#,}" # remove the leading ',' added above - -cmd=( - ${base}/hacking.py - - --ignore="$IGNORE" - - "${files[@]}" -) - -echo -e "\nRunning 'cloudinit' pep8:" -echo "${cmd[@]}" -"${cmd[@]}" +myname=${0##*/} +cmd=( "${myname#run-}" $verbose "${files[@]}" ) +echo "Running: " "${cmd[@]}" 1>&2 +exec "${cmd[@]}" -- cgit v1.2.3