From 89c5936c7c1fb6d172cd0eee9c5f9aa2cd5e2053 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 2 Aug 2015 16:50:47 -0400 Subject: sync with 2.0 trunk on reporting --- cloudinit/reporting/handlers.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 cloudinit/reporting/handlers.py (limited to 'cloudinit/reporting/handlers.py') diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py new file mode 100644 index 00000000..be323f53 --- /dev/null +++ b/cloudinit/reporting/handlers.py @@ -0,0 +1,25 @@ +import abc +import logging + +from cloudinit.registry import DictRegistry + + +class ReportingHandler(object): + + @abc.abstractmethod + def publish_event(self, event): + raise NotImplementedError + + +class LogHandler(ReportingHandler): + """Publishes events to the cloud-init log at the ``INFO`` log level.""" + + def publish_event(self, event): + """Publish an event to the ``INFO`` log level.""" + logger = logging.getLogger( + '.'.join(['cloudinit', 'reporting', event.event_type, event.name])) + logger.info(event.as_string()) + + +available_handlers = DictRegistry() +available_handlers.register_item('log', LogHandler) -- cgit v1.2.3 From 89c564a6fd5ac89869f83541370557e3fa58495c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 2 Aug 2015 17:51:40 -0400 Subject: fix tests from sync change ReportStack to ReportEventStack change default ReportEventStack to be status.SUCCESS instead of None --- bin/cloud-init | 3 +- cloudinit/cloud.py | 2 +- cloudinit/reporting/__init__.py | 91 +++++++++++++++++++++++++++++++++++---- cloudinit/reporting/handlers.py | 7 +++ cloudinit/sources/__init__.py | 2 +- cloudinit/stages.py | 10 ++--- tests/unittests/test_reporting.py | 19 ++++---- 7 files changed, 109 insertions(+), 25 deletions(-) (limited to 'cloudinit/reporting/handlers.py') diff --git a/bin/cloud-init b/bin/cloud-init index d369a806..51253c42 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -637,7 +637,8 @@ def main(): "running single module %s" % args.name) report_on = args.report - args.reporter = reporting.ReportStack( + reporting.add_configuration({'print': {'type': 'print'}}) + args.reporter = reporting.ReportEventStack( rname, rdesc, reporting_enabled=report_on) with args.reporter: return util.log_time( diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index 71eb80eb..a0fb42a3 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -47,7 +47,7 @@ class Cloud(object): self._cfg = cfg self._runners = runners if reporter is None: - reporter = reporting.ReportStack( + reporter = reporting.ReportEventStack( name="unnamed-cloud-reporter", description="unnamed-cloud-reporter", reporting_enabled=False) diff --git a/cloudinit/reporting/__init__.py b/cloudinit/reporting/__init__.py index b0364eec..78dde715 100644 --- a/cloudinit/reporting/__init__.py +++ b/cloudinit/reporting/__init__.py @@ -22,6 +22,15 @@ DEFAULT_CONFIG = { instantiated_handler_registry = DictRegistry() +class _nameset(set): + def __getattr__(self, name): + if name in self: + return name + raise AttributeError("%s not a valid value" % name) + + +status = _nameset(("SUCCESS", "WARN", "FAIL")) + class ReportingEvent(object): """Encapsulation of event formatting.""" @@ -39,17 +48,16 @@ class ReportingEvent(object): class FinishReportingEvent(ReportingEvent): - def __init__(self, name, description, successful=None): + def __init__(self, name, description, result=status.SUCCESS): super(FinishReportingEvent, self).__init__( FINISH_EVENT_TYPE, name, description) - self.successful = successful + self.result = result + if result not in status: + raise ValueError("Invalid result: %s" % result) def as_string(self): - if self.successful is None: - return super(FinishReportingEvent, self).as_string() - success_string = 'success' if self.successful else 'fail' return '{0}: {1}: {2}: {3}'.format( - self.event_type, self.name, success_string, self.description) + self.event_type, self.name, self.result, self.description) def add_configuration(config): @@ -74,12 +82,13 @@ def report_event(event): handler.publish_event(event) -def report_finish_event(event_name, event_description, successful=None): +def report_finish_event(event_name, event_description, + result=status.SUCCESS): """Report a "finish" event. See :py:func:`.report_event` for parameter details. """ - event = FinishReportingEvent(event_name, event_description, successful) + event = FinishReportingEvent(event_name, event_description, result) return report_event(event) @@ -97,4 +106,70 @@ def report_start_event(event_name, event_description): return report_event(event) +class ReportEventStack(object): + def __init__(self, name, description, message=None, parent=None, + reporting_enabled=None, result_on_exception=status.FAIL): + self.parent = parent + self.name = name + self.description = description + self.message = message + self.result_on_exception = result_on_exception + self.result = status.SUCCESS + + # use parents reporting value if not provided + if reporting_enabled is None: + if parent: + reporting_enabled = parent.reporting_enabled + else: + reporting_enabled = True + self.reporting_enabled = reporting_enabled + + if parent: + self.fullname = '/'.join((parent.fullname, name,)) + else: + self.fullname = self.name + self.children = {} + + def __repr__(self): + return ("%s reporting=%s" % (self.fullname, self.reporting_enabled)) + + def __enter__(self): + self.result = status.SUCCESS + if self.reporting_enabled: + report_start_event(self.fullname, self.description) + if self.parent: + self.parent.children[self.name] = (None, None) + return self + + def childrens_finish_info(self): + for cand_result in (status.FAIL, status.WARN): + for name, (value, msg) in self.children.items(): + if value == cand_result: + return (value, "[" + name + "]" + msg) + return (self.result, self.message) + + @property + def message(self): + if self._message is not None: + return self._message + return self.description + + @message.setter + def message(self, value): + self._message = value + + def finish_info(self, exc): + # return tuple of description, and value + if exc: + return (self.result_on_exception, self.message) + return self.childrens_finish_info() + + def __exit__(self, exc_type, exc_value, traceback): + (result, msg) = self.finish_info(exc_value) + if self.parent: + self.parent.children[self.name] = (result, msg) + if self.reporting_enabled: + report_finish_event(self.fullname, msg, result) + + add_configuration(DEFAULT_CONFIG) diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index be323f53..1d5ca524 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -21,5 +21,12 @@ class LogHandler(ReportingHandler): logger.info(event.as_string()) +class StderrHandler(ReportingHandler): + def publish_event(self, event): + #sys.stderr.write(event.as_string() + "\n") + print(event.as_string()) + + available_handlers = DictRegistry() available_handlers.register_item('log', LogHandler) +available_handlers.register_item('print', StderrHandler) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 3b48f173..d07cf1fa 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -254,7 +254,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter): LOG.debug("Searching for %s data source in: %s", mode, ds_names) for name, cls in zip(ds_names, ds_list): - myrep = reporting.ReportStack( + myrep = reporting.ReportEventStack( name="search-%s-%s" % (mode, name.replace("DataSource", "")), description="searching for %s data from %s" % (mode, name), message = "no %s data found from %s" % (mode, name), diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 8c79ae4e..42989bb4 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -67,7 +67,7 @@ class Init(object): self.datasource = NULL_DATA_SOURCE if reporter is None: - reporter = reporting.ReportStack( + reporter = reporting.ReportEventStack( name="init-reporter", description="init-desc", reporting_enabled=False) self.reporter = reporter @@ -242,7 +242,7 @@ class Init(object): if self.datasource is not NULL_DATA_SOURCE: return self.datasource - with reporting.ReportStack( + with reporting.ReportEventStack( name="check-cache", description="attempting to read from cache", parent=self.reporter) as myrep: ds = self._restore_from_cache() @@ -508,11 +508,11 @@ class Init(object): def consume_data(self, frequency=PER_INSTANCE): # Consume the userdata first, because we need want to let the part # handlers run first (for merging stuff) - with reporting.ReportStack( + with reporting.ReportEventStack( "consume-user-data", "reading and applying user-data", parent=self.reporter): self._consume_userdata(frequency) - with reporting.ReportStack( + with reporting.ReportEventStack( "consume-vendor-data", "reading and applying vendor-data", parent=self.reporter): self._consume_userdata(frequency) @@ -705,7 +705,7 @@ class Modules(object): run_name = "config-%s" % (name) desc="running %s with frequency %s" % (run_name, freq) - myrep = reporting.ReportStack( + myrep = reporting.ReportEventStack( name=run_name, description=desc, parent=self.reporter) with myrep: diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 5700118f..4f4cf3a4 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -32,7 +32,7 @@ class TestReportStartEvent(TestCase): class TestReportFinishEvent(TestCase): - def _report_finish_event(self, result=None): + def _report_finish_event(self, result=reporting.status.SUCCESS): event_name, event_description = 'my_test_event', 'my description' reporting.report_finish_event( event_name, event_description, result=result) @@ -95,31 +95,32 @@ class TestReportingHandler(TestCase): def test_no_default_publish_event_implementation(self): self.assertRaises(NotImplementedError, - reporting.ReportingHandler().publish_event, None) + reporting.handlers.ReportingHandler().publish_event, + None) class TestLogHandler(TestCase): - @mock.patch.object(reporting.logging, 'getLogger') + @mock.patch.object(reporting.handlers.logging, 'getLogger') def test_appropriate_logger_used(self, getLogger): event_type, event_name = 'test_type', 'test_name' event = reporting.ReportingEvent(event_type, event_name, 'description') - reporting.LogHandler().publish_event(event) + reporting.handlers.LogHandler().publish_event(event) self.assertEqual( [mock.call( 'cloudinit.reporting.{0}.{1}'.format(event_type, event_name))], getLogger.call_args_list) - @mock.patch.object(reporting.logging, 'getLogger') + @mock.patch.object(reporting.handlers.logging, 'getLogger') def test_single_log_message_at_info_published(self, getLogger): event = reporting.ReportingEvent('type', 'name', 'description') - reporting.LogHandler().publish_event(event) + reporting.handlers.LogHandler().publish_event(event) self.assertEqual(1, getLogger.return_value.info.call_count) - @mock.patch.object(reporting.logging, 'getLogger') + @mock.patch.object(reporting.handlers.logging, 'getLogger') def test_log_message_uses_event_as_string(self, getLogger): event = reporting.ReportingEvent('type', 'name', 'description') - reporting.LogHandler().publish_event(event) + reporting.handlers.LogHandler().publish_event(event) self.assertIn(event.as_string(), getLogger.return_value.info.call_args[0][0]) @@ -130,7 +131,7 @@ class TestDefaultRegisteredHandler(TestCase): registered_items = ( reporting.instantiated_handler_registry.registered_items) for _, item in registered_items.items(): - if isinstance(item, reporting.LogHandler): + if isinstance(item, reporting.handlers.LogHandler): break else: self.fail('No reporting LogHandler registered by default.') -- cgit v1.2.3 From 5585b397cfb4ba397e9cfba3d86e3d10af20eb71 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 4 Aug 2015 22:01:27 -0500 Subject: fix pep8 --- bin/cloud-init | 3 ++- cloudinit/reporting/handlers.py | 7 ------- cloudinit/sources/__init__.py | 2 +- cloudinit/stages.py | 10 ++++++---- tests/unittests/test_reporting.py | 6 ++++-- 5 files changed, 13 insertions(+), 15 deletions(-) (limited to 'cloudinit/reporting/handlers.py') diff --git a/bin/cloud-init b/bin/cloud-init index 51253c42..40cdbb06 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -628,7 +628,8 @@ def main(): if args.local: rname, rdesc = ("init-local", "searching for local datasources") else: - rname, rdesc = ("init-network", "searching for network datasources") + rname, rdesc = ("init-network", + "searching for network datasources") elif name == "modules": rname, rdesc = ("modules-%s" % args.mode, "running modules for %s" % args.mode) diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 1d5ca524..be323f53 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -21,12 +21,5 @@ class LogHandler(ReportingHandler): logger.info(event.as_string()) -class StderrHandler(ReportingHandler): - def publish_event(self, event): - #sys.stderr.write(event.as_string() + "\n") - print(event.as_string()) - - available_handlers = DictRegistry() available_handlers.register_item('log', LogHandler) -available_handlers.register_item('print', StderrHandler) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index cf50c1fb..838cd198 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -257,7 +257,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter): myrep = reporting.ReportEventStack( name="search-%s" % name.replace("DataSource", ""), description="searching for %s data from %s" % (mode, name), - message = "no %s data found from %s" % (mode, name), + message="no %s data found from %s" % (mode, name), parent=reporter) try: with myrep: diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 7b489b9f..d300709d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -243,7 +243,8 @@ class Init(object): return self.datasource with reporting.ReportEventStack( - name="check-cache", description="attempting to read from cache", + name="check-cache", + description="attempting to read from cache", parent=self.reporter) as myrep: ds = self._restore_from_cache() if ds: @@ -708,17 +709,18 @@ class Modules(object): # This name will affect the semaphore name created run_name = "config-%s" % (name) - desc="running %s with frequency %s" % (run_name, freq) + desc = "running %s with frequency %s" % (run_name, freq) myrep = reporting.ReportEventStack( name=run_name, description=desc, parent=self.reporter) with myrep: - ran, _r = cc.run(run_name, mod.handle, func_args, freq=freq) + ran, _r = cc.run(run_name, mod.handle, func_args, + freq=freq) if ran: myrep.message = "%s ran successfully" % run_name else: myrep.message = "%s previously ran" % run_name - + except Exception as e: util.logexc(LOG, "Running module %s (%s) failed", name, mod) failures.append((name, e)) diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 4f4cf3a4..ddfac541 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -51,7 +51,8 @@ class TestReportFinishEvent(TestCase): self, instantiated_handler_registry): event_name, event_description = self._report_finish_event() expected_string_representation = ': '.join( - ['finish', event_name, reporting.status.SUCCESS, event_description]) + ['finish', event_name, reporting.status.SUCCESS, + event_description]) self.assertHandlersPassedObjectWithAsString( instantiated_handler_registry.registered_items, expected_string_representation) @@ -63,7 +64,8 @@ class TestReportFinishEvent(TestCase): event_name, event_description = self._report_finish_event( result=reporting.status.SUCCESS) expected_string_representation = ': '.join( - ['finish', event_name, reporting.status.SUCCESS, event_description]) + ['finish', event_name, reporting.status.SUCCESS, + event_description]) self.assertHandlersPassedObjectWithAsString( instantiated_handler_registry.registered_items, expected_string_representation) -- cgit v1.2.3 From 6fdb23b6cbc8de14ebcffc17e9e49342b7bf193d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 6 Aug 2015 18:19:46 -0500 Subject: sync with cloudinit 2.0 for registry and reporting --- cloudinit/registry.py | 14 +++++++++++ cloudinit/reporting/__init__.py | 29 +++++++++++++++++++--- cloudinit/reporting/handlers.py | 15 +++++++++++- tests/unittests/test_reporting.py | 51 ++++++++++++++++++++++++++++++++------- 4 files changed, 95 insertions(+), 14 deletions(-) (limited to 'cloudinit/reporting/handlers.py') diff --git a/cloudinit/registry.py b/cloudinit/registry.py index 46cf0585..04368ddf 100644 --- a/cloudinit/registry.py +++ b/cloudinit/registry.py @@ -1,3 +1,7 @@ +# Copyright 2015 Canonical Ltd. +# This file is part of cloud-init. See LICENCE file for license information. +# +# vi: ts=4 expandtab import copy @@ -5,6 +9,9 @@ class DictRegistry(object): """A simple registry for a mapping of objects.""" def __init__(self): + self.reset() + + def reset(self): self._items = {} def register_item(self, key, item): @@ -14,6 +21,13 @@ class DictRegistry(object): 'Item already registered with key {0}'.format(key)) self._items[key] = item + def unregister_item(self, key, force=True): + """Remove item from the registry.""" + if key in self._items: + del self._items[key] + elif not force: + raise KeyError("%s: key not present to unregister" % key) + @property def registered_items(self): """All the items that have been registered. diff --git a/cloudinit/reporting/__init__.py b/cloudinit/reporting/__init__.py index 2b92ab58..d0bc14e3 100644 --- a/cloudinit/reporting/__init__.py +++ b/cloudinit/reporting/__init__.py @@ -20,8 +20,6 @@ DEFAULT_CONFIG = { 'logging': {'type': 'log'}, } -instantiated_handler_registry = DictRegistry() - class _nameset(set): def __getattr__(self, name): @@ -46,6 +44,11 @@ class ReportingEvent(object): return '{0}: {1}: {2}'.format( self.event_type, self.name, self.description) + def as_dict(self): + """The event represented as a dictionary.""" + return {'name': self.name, 'description': self.description, + 'event_type': self.event_type} + class FinishReportingEvent(ReportingEvent): @@ -60,9 +63,26 @@ class FinishReportingEvent(ReportingEvent): return '{0}: {1}: {2}: {3}'.format( self.event_type, self.name, self.result, self.description) + def as_dict(self): + """The event represented as json friendly.""" + data = super(FinishReportingEvent, self).as_dict() + data['result'] = self.result + return data + -def add_configuration(config): +def update_configuration(config): + """Update the instanciated_handler_registry. + + :param config: + The dictionary containing changes to apply. If a key is given + with a False-ish value, the registered handler matching that name + will be unregistered. + """ for handler_name, handler_config in config.items(): + if not handler_config: + instantiated_handler_registry.unregister_item( + handler_name, force=True) + continue handler_config = handler_config.copy() cls = available_handlers.registered_items[handler_config.pop('type')] instance = cls(**handler_config) @@ -214,4 +234,5 @@ class ReportEventStack(object): report_finish_event(self.fullname, msg, result) -add_configuration(DEFAULT_CONFIG) +instantiated_handler_registry = DictRegistry() +update_configuration(DEFAULT_CONFIG) diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index be323f53..86cbe3c3 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -1,14 +1,27 @@ +# vi: ts=4 expandtab + import abc import logging +import oauthlib.oauth1 as oauth1 + +import six from cloudinit.registry import DictRegistry +from cloudinit import url_helper +from cloudinit import util +@six.add_metaclass(abc.ABCMeta) class ReportingHandler(object): + """Base class for report handlers. + + Implement :meth:`~publish_event` for controlling what + the handler does with an event. + """ @abc.abstractmethod def publish_event(self, event): - raise NotImplementedError + """Publish an event to the ``INFO`` log level.""" class LogHandler(ReportingHandler): diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index ffeb55d2..1a4ee8c4 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -4,6 +4,7 @@ # vi: ts=4 expandtab from cloudinit import reporting +from cloudinit.reporting import handlers from .helpers import (mock, TestCase) @@ -95,13 +96,29 @@ class TestReportingEvent(TestCase): [event_type, name, description]) self.assertEqual(expected_string_representation, event.as_string()) + def test_as_dict(self): + event_type, name, desc = 'test_type', 'test_name', 'test_desc' + event = reporting.ReportingEvent(event_type, name, desc) + self.assertEqual( + {'event_type': event_type, 'name': name, 'description': desc}, + event.as_dict()) + + +class TestFinishReportingEvent(TestCase): + def test_as_has_result(self): + result = reporting.status.SUCCESS + name, desc = 'test_name', 'test_desc' + event = reporting.FinishReportingEvent(name, desc, result) + ret = event.as_dict() + self.assertTrue('result' in ret) + self.assertEqual(ret['result'], result) + -class TestReportingHandler(TestCase): +class TestBaseReportingHandler(TestCase): - def test_no_default_publish_event_implementation(self): - self.assertRaises(NotImplementedError, - reporting.handlers.ReportingHandler().publish_event, - None) + def test_base_reporting_handler_is_abstract(self): + regexp = r".*abstract.*publish_event.*" + self.assertRaisesRegexp(TypeError, regexp, handlers.ReportingHandler) class TestLogHandler(TestCase): @@ -147,7 +164,7 @@ class TestReportingConfiguration(TestCase): @mock.patch.object(reporting, 'instantiated_handler_registry') def test_empty_configuration_doesnt_add_handlers( self, instantiated_handler_registry): - reporting.add_configuration({}) + reporting.update_configuration({}) self.assertEqual( 0, instantiated_handler_registry.register_item.call_count) @@ -159,7 +176,7 @@ class TestReportingConfiguration(TestCase): handler_cls = mock.Mock() available_handlers.registered_items = {handler_type_name: handler_cls} handler_name = 'my_test_handler' - reporting.add_configuration( + reporting.update_configuration( {handler_name: {'type': handler_type_name}}) self.assertEqual( {handler_name: handler_cls.return_value}, @@ -177,7 +194,7 @@ class TestReportingConfiguration(TestCase): handler_config = extra_kwargs.copy() handler_config.update({'type': handler_type_name}) handler_name = 'my_test_handler' - reporting.add_configuration({handler_name: handler_config}) + reporting.update_configuration({handler_name: handler_config}) self.assertEqual( handler_cls.return_value, reporting.instantiated_handler_registry.registered_items[ @@ -194,9 +211,25 @@ class TestReportingConfiguration(TestCase): available_handlers.registered_items = {handler_type_name: handler_cls} handler_config = {'type': handler_type_name, 'foo': 'bar'} expected_handler_config = handler_config.copy() - reporting.add_configuration({'my_test_handler': handler_config}) + reporting.update_configuration({'my_test_handler': handler_config}) self.assertEqual(expected_handler_config, handler_config) + @mock.patch.object( + reporting, 'instantiated_handler_registry', reporting.DictRegistry()) + @mock.patch.object(reporting, 'available_handlers') + def test_handlers_removed_if_falseish_specified(self, available_handlers): + handler_type_name = 'test_handler' + handler_cls = mock.Mock() + available_handlers.registered_items = {handler_type_name: handler_cls} + handler_name = 'my_test_handler' + reporting.update_configuration( + {handler_name: {'type': handler_type_name}}) + self.assertEqual( + 1, len(reporting.instantiated_handler_registry.registered_items)) + reporting.update_configuration({handler_name: None}) + self.assertEqual( + 0, len(reporting.instantiated_handler_registry.registered_items)) + class TestReportingEventStack(TestCase): @mock.patch('cloudinit.reporting.report_finish_event') -- cgit v1.2.3 From ebd393e56ba21f8a84571dff499e6d6fb6852042 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 6 Aug 2015 18:34:57 -0500 Subject: tests pass --- bin/cloud-init | 10 +++ cloudinit/reporting/handlers.py | 28 +++++++ cloudinit/sources/DataSourceMAAS.py | 88 +++++----------------- cloudinit/url_helper.py | 142 ++++++++++++++++++++++++++++++++++-- cloudinit/util.py | 3 +- 5 files changed, 196 insertions(+), 75 deletions(-) (limited to 'cloudinit/reporting/handlers.py') diff --git a/bin/cloud-init b/bin/cloud-init index 40cdbb06..ad2e624a 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -137,6 +137,11 @@ def run_module_section(mods, action_name, section): return failures +def apply_reporting_cfg(cfg): + reporting.reset_configuration() + reporting.update_configuration(cfg.get('reporting'), {}) + + def main_init(name, args): deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] if args.local: @@ -191,6 +196,7 @@ def main_init(name, args): " longer be active shortly")) logging.resetLogging() logging.setupLogging(init.cfg) + apply_reporting_cfg(init.cfg) # Any log usage prior to setupLogging above did not have local user log # config applied. We send the welcome message now, as stderr/out have @@ -283,6 +289,8 @@ def main_init(name, args): util.logexc(LOG, "Consuming user data failed!") return (init.datasource, ["Consuming user data failed!"]) + apply_reporting_cfg(init.cfg) + # Stage 8 - re-read and apply relevant cloud-config to include user-data mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) # Stage 9 @@ -343,6 +351,7 @@ def main_modules(action_name, args): " longer be active shortly")) logging.resetLogging() logging.setupLogging(mods.cfg) + apply_reporting_cfg(init.cfg) # now that logging is setup and stdout redirected, send welcome welcome(name, msg=w_msg) @@ -405,6 +414,7 @@ def main_single(name, args): " longer be active shortly")) logging.resetLogging() logging.setupLogging(mods.cfg) + apply_reporting_cfg(init.cfg) # now that logging is setup and stdout redirected, send welcome welcome(name, msg=w_msg) diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 86cbe3c3..d8f69641 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -34,5 +34,33 @@ class LogHandler(ReportingHandler): logger.info(event.as_string()) +class WebHookHandler(ReportingHandler): + def __init__(self, endpoint, consumer_key=None, token_key=None, + token_secret=None, consumer_secret=None, timeout=None, + retries=None): + super(WebHookHandler, self).__init__() + + if any(consumer_key, token_key, token_secret, consumer_secret): + self.oauth_helper = url_helper.OauthHelper( + consumer_key=consumer_key, token_key=token_key, + token_secret=token_secret, consumer_secret=consumer_secret) + else: + self.oauth_helper = None + self.endpoint = endpoint + self.timeout = timeout + self.retries = retries + self.ssl_details = util.fetch_ssl_details() + + def publish_event(self, event): + if self.oauth_helper: + readurl = self.oauth_helper.readurl + else: + readurl = url_helper.readurl + return readurl( + self.endpoint, data=event.as_dict(), + timeout=self.timeout, + retries=self.retries, ssl_details=self.ssl_details) + + available_handlers = DictRegistry() available_handlers.register_item('log', LogHandler) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index c1a0eb61..279da238 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -52,7 +52,20 @@ class DataSourceMAAS(sources.DataSource): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.base_url = None self.seed_dir = os.path.join(paths.seed_dir, 'maas') - self.oauth_clockskew = None + self.oauth_helper = self._get_helper() + + def _get_helper(self): + mcfg = self.ds_cfg + # If we are missing token_key, token_secret or consumer_key + # then just do non-authed requests + for required in ('token_key', 'token_secret', 'consumer_key'): + if required not in mcfg: + return url_helper.OauthUrlHelper() + + return url_helper.OauthHelper( + consumer_key=mcfg['consumer_key'], token_key=mcfg['token_key'], + token_secret=mcfg['token_secret'], + consumer_secret=mcfg.get('consumer_secret')) def __str__(self): root = sources.DataSource.__str__(self) @@ -84,9 +97,9 @@ class DataSourceMAAS(sources.DataSource): self.base_url = url - (userdata, metadata) = read_maas_seed_url(self.base_url, - self._md_headers, - paths=self.paths) + (userdata, metadata) = read_maas_seed_url( + self.base_url, self.oauth_helper.md_headers, + paths=self.paths) self.userdata_raw = userdata self.metadata = metadata return True @@ -94,31 +107,8 @@ class DataSourceMAAS(sources.DataSource): util.logexc(LOG, "Failed fetching metadata from url %s", url) return False - def _md_headers(self, url): - mcfg = self.ds_cfg - - # If we are missing token_key, token_secret or consumer_key - # then just do non-authed requests - for required in ('token_key', 'token_secret', 'consumer_key'): - if required not in mcfg: - return {} - - consumer_secret = mcfg.get('consumer_secret', "") - - timestamp = None - if self.oauth_clockskew: - timestamp = int(time.time()) + self.oauth_clockskew - - return oauth_headers(url=url, - consumer_key=mcfg['consumer_key'], - token_key=mcfg['token_key'], - token_secret=mcfg['token_secret'], - consumer_secret=consumer_secret, - timestamp=timestamp) - def wait_for_metadata_service(self, url): mcfg = self.ds_cfg - max_wait = 120 try: max_wait = int(mcfg.get("max_wait", max_wait)) @@ -138,10 +128,8 @@ class DataSourceMAAS(sources.DataSource): starttime = time.time() check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) urls = [check_url] - url = url_helper.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, - exception_cb=self._except_cb, - headers_cb=self._md_headers) + url = self.oauth_helper.wait_for_url( + urls=urls, max_wait=max_wait, timeout=timeout) if url: LOG.debug("Using metadata source: '%s'", url) @@ -151,26 +139,6 @@ class DataSourceMAAS(sources.DataSource): return bool(url) - def _except_cb(self, msg, exception): - if not (isinstance(exception, url_helper.UrlError) and - (exception.code == 403 or exception.code == 401)): - return - - if 'date' not in exception.headers: - LOG.warn("Missing header 'date' in %s response", exception.code) - return - - date = exception.headers['date'] - try: - ret_time = time.mktime(parsedate(date)) - except Exception as e: - LOG.warn("Failed to convert datetime '%s': %s", date, e) - return - - self.oauth_clockskew = int(ret_time - time.time()) - LOG.warn("Setting oauth clockskew to %d", self.oauth_clockskew) - return - def read_maas_seed_dir(seed_d): """ @@ -280,24 +248,6 @@ def check_seed_contents(content, seed): return (userdata, md) -def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret, - timestamp=None): - if timestamp: - timestamp = str(timestamp) - else: - timestamp = None - - client = oauth1.Client( - consumer_key, - client_secret=consumer_secret, - resource_owner_key=token_key, - resource_owner_secret=token_secret, - signature_method=oauth1.SIGNATURE_PLAINTEXT, - timestamp=timestamp) - uri, signed_headers, body = client.sign(url) - return signed_headers - - class MAASSeedDirNone(Exception): pass diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 0e65f431..2141cdc5 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -25,6 +25,10 @@ import time import six import requests +import oauthlib.oauth1 as oauth1 +import os +import json +from functools import partial from requests import exceptions from six.moves.urllib.parse import ( @@ -147,13 +151,14 @@ class UrlResponse(object): class UrlError(IOError): - def __init__(self, cause, code=None, headers=None): + def __init__(self, cause, code=None, headers=None, url=None): IOError.__init__(self, str(cause)) self.cause = cause self.code = code self.headers = headers if self.headers is None: self.headers = {} + self.url = url def _get_ssl_args(url, ssl_details): @@ -247,9 +252,10 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, and hasattr(e, 'response') # This appeared in v 0.10.8 and hasattr(e.response, 'status_code')): excps.append(UrlError(e, code=e.response.status_code, - headers=e.response.headers)) + headers=e.response.headers, + url=url)) else: - excps.append(UrlError(e)) + excps.append(UrlError(e, url=url)) if SSL_ENABLED and isinstance(e, exceptions.SSLError): # ssl exceptions are not going to get fixed by waiting a # few seconds @@ -333,11 +339,11 @@ def wait_for_url(urls, max_wait=None, timeout=None, if not response.contents: reason = "empty response [%s]" % (response.code) url_exc = UrlError(ValueError(reason), code=response.code, - headers=response.headers) + headers=response.headers, url=url) elif not response.ok(): reason = "bad status code [%s]" % (response.code) url_exc = UrlError(ValueError(reason), code=response.code, - headers=response.headers) + headers=response.headers, url=url) else: return url except UrlError as e: @@ -368,3 +374,129 @@ def wait_for_url(urls, max_wait=None, timeout=None, time.sleep(sleep_time) return False + + +class OauthUrlHelper(object): + def __init__(self, consumer_key=None, token_key=None, + token_secret=None, consumer_secret=None, + skew_data_file="/run/oauth_skew.json"): + self.consumer_key = consumer_key + self.consumer_secret = consumer_secret or "" + self.token_key = token_key + self.token_secret = token_secret + self.skew_data_file = skew_data_file + self.skew_data = {} + self._do_oauth = True + self.skew_change_limit = 5 + required = (self.token_key, self.token_secret, self.consumer_key) + if not any(required): + self._do_oauth = False + elif not all(required): + raise ValueError("all or none of token_key, token_secret, or " + "consumer_key can be set") + + self.skew_data = self.read_skew_file() + + def read_skew_file(self): + if self.skew_data_file and os.path.isfile(self.skew_data_file): + with open(self.skew_data_file, mode="r") as fp: + return json.load(fp.read()) + return None + + def update_skew_file(self, host, value): + # this is not atomic + cur = self.read_skew_file() + if cur is None or not self.skew_data_file: + return + cur[host] = value + with open(self.skew_data_file, mode="w") as fp: + fp.write(json.dumps(cur)) + + def exception_cb(self, msg, exception): + if not (isinstance(exception, UrlError) and + (exception.code == 403 or exception.code == 401)): + return + + if 'date' not in exception.headers: + LOG.warn("Missing header 'date' in %s response", exception.code) + return + + date = exception.headers['date'] + try: + ret_time = time.mktime(parsedate(date)) + except Exception as e: + LOG.warn("Failed to convert datetime '%s': %s", date, e) + return + + host = urlparse(exception.url).netloc + skew = int(ret_time - time.time()) + old_skew = self.skew_data.get(host) + if abs(old_skew - skew) > self.skew_change_limit: + self.update_skew_file(host, skew) + LOG.warn("Setting oauth clockskew for %s to %d", + host, skew) + skew_data[host] = skew + + return + + def headers_cb(self, url): + if not self._do_oauth: + return {} + + timestamp = None + host = urlparse(url).netloc + if host in self.skew_data: + timestamp = int(time.time()) + self.skew_data[host] + + return oauth_headers( + url=url, consumer_key=self.consumer_key, + token_key=self.token_key, token_secret=self.token_secret, + consumer_secret=self.consumer_secret, timestamp=timestamp) + + def _wrapped(self, wrapped_func, args, kwargs): + kwargs['headers_cb'] = partial( + self._headers_cb, kwargs.get('headers_cb')) + kwargs['exception_cb'] = partial( + self._exception_cb, kwargs.get('exception_cb')) + return wrapped_func(*args, **kwargs) + + def wait_for_url(self, *args, **kwargs): + return self._wrapped(wait_for_url, args, kwargs) + + def readurl(self, *args, **kwargs): + return self._wrapped(readurl, args, kwargs) + + def _exception_cb(self, extra_exception_cb, url, msg, exception): + ret = None + try: + if extra_exception_cb: + ret = extra_exception_cb(msg, exception) + finally: + self.exception_cb(self, msg, exception) + return ret + + def _headers_cb(self, extra_headers_cb, url): + headers = {} + if extra_headers_cb: + headers = extra_headers_cb(url) + if headers: + headers.update(self.headers_cb(url)) + return headers + + +def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret, + timestamp=None): + if timestamp: + timestamp = str(timestamp) + else: + timestamp = None + + client = oauth1.Client( + consumer_key, + client_secret=consumer_secret, + resource_owner_key=token_key, + resource_owner_secret=token_secret, + signature_method=oauth1.SIGNATURE_PLAINTEXT, + timestamp=timestamp) + uri, signed_headers, body = client.sign(url) + return signed_headers diff --git a/cloudinit/util.py b/cloudinit/util.py index 02ba654a..09e583f5 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -782,7 +782,8 @@ def read_file_or_url(url, timeout=5, retries=10, code = e.errno if e.errno == errno.ENOENT: code = url_helper.NOT_FOUND - raise url_helper.UrlError(cause=e, code=code, headers=None) + raise url_helper.UrlError(cause=e, code=code, headers=None, + url=url) return url_helper.FileResponse(file_path, contents=contents) else: return url_helper.readurl(url, -- cgit v1.2.3 From fc5fc6e476059327d4063f165170cdde01db4100 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 6 Aug 2015 23:51:17 -0500 Subject: add the webhook handler --- bin/cloud-init | 6 +++--- cloudinit/reporting/__init__.py | 11 +++++++++-- cloudinit/reporting/handlers.py | 15 +++++++++++---- 3 files changed, 23 insertions(+), 9 deletions(-) (limited to 'cloudinit/reporting/handlers.py') diff --git a/bin/cloud-init b/bin/cloud-init index ad2e624a..86780408 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -138,8 +138,8 @@ def run_module_section(mods, action_name, section): def apply_reporting_cfg(cfg): - reporting.reset_configuration() - reporting.update_configuration(cfg.get('reporting'), {}) + if cfg.get('reporting'): + reporting.update_configuration(cfg.get('reporting')) def main_init(name, args): @@ -648,7 +648,7 @@ def main(): "running single module %s" % args.name) report_on = args.report - reporting.add_configuration({'print': {'type': 'print'}}) + reporting.update_configuration({'print': {'type': 'print'}}) args.reporter = reporting.ReportEventStack( rname, rdesc, reporting_enabled=report_on) with args.reporter: diff --git a/cloudinit/reporting/__init__.py b/cloudinit/reporting/__init__.py index d0bc14e3..b9d4f679 100644 --- a/cloudinit/reporting/__init__.py +++ b/cloudinit/reporting/__init__.py @@ -9,8 +9,8 @@ The reporting framework is intended to allow all parts of cloud-init to report events in a structured manner. """ -from cloudinit.registry import DictRegistry -from cloudinit.reporting.handlers import available_handlers +from ..registry import DictRegistry +from ..reporting.handlers import available_handlers FINISH_EVENT_TYPE = 'finish' @@ -18,6 +18,7 @@ START_EVENT_TYPE = 'start' DEFAULT_CONFIG = { 'logging': {'type': 'log'}, + 'print': {'type': 'print'}, } @@ -83,8 +84,14 @@ def update_configuration(config): instantiated_handler_registry.unregister_item( handler_name, force=True) continue + registered = instantiated_handler_registry.registered_items handler_config = handler_config.copy() cls = available_handlers.registered_items[handler_config.pop('type')] + if (handler_name in registered and + (registered[handler_name] == handler_config)): + continue + else: + instantiated_handler_registry.unregister_item(handler_name) instance = cls(**handler_config) instantiated_handler_registry.register_item(handler_name, instance) diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index d8f69641..a962edae 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -6,9 +6,8 @@ import oauthlib.oauth1 as oauth1 import six -from cloudinit.registry import DictRegistry -from cloudinit import url_helper -from cloudinit import util +from ..registry import DictRegistry +from .. import (url_helper, util) @six.add_metaclass(abc.ABCMeta) @@ -34,13 +33,19 @@ class LogHandler(ReportingHandler): logger.info(event.as_string()) +class PrintHandler(ReportingHandler): + def publish_event(self, event): + """Publish an event to the ``INFO`` log level.""" + print(event.as_string()) + + class WebHookHandler(ReportingHandler): def __init__(self, endpoint, consumer_key=None, token_key=None, token_secret=None, consumer_secret=None, timeout=None, retries=None): super(WebHookHandler, self).__init__() - if any(consumer_key, token_key, token_secret, consumer_secret): + if any([consumer_key, token_key, token_secret, consumer_secret]): self.oauth_helper = url_helper.OauthHelper( consumer_key=consumer_key, token_key=token_key, token_secret=token_secret, consumer_secret=consumer_secret) @@ -64,3 +69,5 @@ class WebHookHandler(ReportingHandler): available_handlers = DictRegistry() available_handlers.register_item('log', LogHandler) +available_handlers.register_item('print', PrintHandler) +available_handlers.register_item('webhook', WebHookHandler) -- cgit v1.2.3 From 89b381f01c727c8fb00724eb28bf98eafd97dbb4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 7 Aug 2015 00:45:11 -0500 Subject: seems functional in test --- cloudinit/reporting/handlers.py | 2 +- cloudinit/url_helper.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) (limited to 'cloudinit/reporting/handlers.py') diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index a962edae..eecd0a96 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -46,7 +46,7 @@ class WebHookHandler(ReportingHandler): super(WebHookHandler, self).__init__() if any([consumer_key, token_key, token_secret, consumer_secret]): - self.oauth_helper = url_helper.OauthHelper( + self.oauth_helper = url_helper.OauthUrlHelper( consumer_key=consumer_key, token_key=token_key, token_secret=token_secret, consumer_secret=consumer_secret) else: diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 2141cdc5..e598661f 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -385,7 +385,6 @@ class OauthUrlHelper(object): self.token_key = token_key self.token_secret = token_secret self.skew_data_file = skew_data_file - self.skew_data = {} self._do_oauth = True self.skew_change_limit = 5 required = (self.token_key, self.token_secret, self.consumer_key) @@ -445,7 +444,7 @@ class OauthUrlHelper(object): timestamp = None host = urlparse(url).netloc - if host in self.skew_data: + if self.skew_data and host in self.skew_data: timestamp = int(time.time()) + self.skew_data[host] return oauth_headers( @@ -466,21 +465,20 @@ class OauthUrlHelper(object): def readurl(self, *args, **kwargs): return self._wrapped(readurl, args, kwargs) - def _exception_cb(self, extra_exception_cb, url, msg, exception): + def _exception_cb(self, extra_exception_cb, msg, exception): ret = None try: if extra_exception_cb: ret = extra_exception_cb(msg, exception) finally: - self.exception_cb(self, msg, exception) + self.exception_cb(msg, exception) return ret def _headers_cb(self, extra_headers_cb, url): headers = {} if extra_headers_cb: headers = extra_headers_cb(url) - if headers: - headers.update(self.headers_cb(url)) + headers.update(self.headers_cb(url)) return headers -- cgit v1.2.3 From 53f35028af55b06c19f409d6081aa766607f22a8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 7 Aug 2015 10:15:10 -0500 Subject: catch exception in webhook, adjust logging to use cloud-init logging --- cloudinit/reporting/handlers.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'cloudinit/reporting/handlers.py') diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index eecd0a96..9cf8bd2b 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -1,13 +1,15 @@ # vi: ts=4 expandtab import abc -import logging import oauthlib.oauth1 as oauth1 - import six from ..registry import DictRegistry from .. import (url_helper, util) +from .. import log as logging + + +LOG = logging.getLogger(__name__) @six.add_metaclass(abc.ABCMeta) @@ -61,10 +63,13 @@ class WebHookHandler(ReportingHandler): readurl = self.oauth_helper.readurl else: readurl = url_helper.readurl - return readurl( - self.endpoint, data=event.as_dict(), - timeout=self.timeout, - retries=self.retries, ssl_details=self.ssl_details) + try: + return readurl( + self.endpoint, data=event.as_dict(), + timeout=self.timeout, + retries=self.retries, ssl_details=self.ssl_details) + except: + LOG.warn("failed posting event: %s" % event.as_string()) available_handlers = DictRegistry() -- cgit v1.2.3 From 95bfe5d5150e2bf0a26dd1b97578c4fd04152365 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 7 Aug 2015 14:44:00 -0500 Subject: add doc, remove some debug / print statements. --- bin/cloud-init | 1 - cloudinit/reporting/__init__.py | 1 - cloudinit/reporting/handlers.py | 16 ++++++++++++++-- doc/examples/cloud-config-reporting.txt | 17 +++++++++++++++++ 4 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 doc/examples/cloud-config-reporting.txt (limited to 'cloudinit/reporting/handlers.py') diff --git a/bin/cloud-init b/bin/cloud-init index 86780408..1f64461e 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -648,7 +648,6 @@ def main(): "running single module %s" % args.name) report_on = args.report - reporting.update_configuration({'print': {'type': 'print'}}) args.reporter = reporting.ReportEventStack( rname, rdesc, reporting_enabled=report_on) with args.reporter: diff --git a/cloudinit/reporting/__init__.py b/cloudinit/reporting/__init__.py index a3b8332f..e23fab32 100644 --- a/cloudinit/reporting/__init__.py +++ b/cloudinit/reporting/__init__.py @@ -18,7 +18,6 @@ START_EVENT_TYPE = 'start' DEFAULT_CONFIG = { 'logging': {'type': 'log'}, - 'print': {'type': 'print'}, } diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 9cf8bd2b..1343311f 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -28,17 +28,29 @@ class ReportingHandler(object): class LogHandler(ReportingHandler): """Publishes events to the cloud-init log at the ``INFO`` log level.""" + def __init__(self, level="DEBUG"): + super(LogHandler, self).__init__() + if isinstance(level, int): + pass + else: + input_level = level + try: + level = gettattr(logging, level.upper()) + except: + LOG.warn("invalid level '%s', using WARN", input_level) + level = logging.WARN + self.level = level + def publish_event(self, event): """Publish an event to the ``INFO`` log level.""" logger = logging.getLogger( '.'.join(['cloudinit', 'reporting', event.event_type, event.name])) - logger.info(event.as_string()) + logger.log(self.level, event.as_string()) class PrintHandler(ReportingHandler): def publish_event(self, event): """Publish an event to the ``INFO`` log level.""" - print(event.as_string()) class WebHookHandler(ReportingHandler): diff --git a/doc/examples/cloud-config-reporting.txt b/doc/examples/cloud-config-reporting.txt new file mode 100644 index 00000000..ee00078f --- /dev/null +++ b/doc/examples/cloud-config-reporting.txt @@ -0,0 +1,17 @@ +#cloud-config +## +## The following sets up 2 reporting end points. +## A 'webhook' and a 'log' type. +## It also disables the built in default 'log' +reporting: + smtest: + type: webhook + endpoint: "http://myhost:8000/" + consumer_key: "ckey_foo" + consumer_secret: "csecret_foo" + token_key: "tkey_foo" + token_secret: "tkey_foo" + smlogger: + type: log + level: WARN + log: null -- cgit v1.2.3 From b39070772aba62d68fea14603b8d657bbb529d5e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 7 Aug 2015 16:06:36 -0500 Subject: reporting: fix logging reproter and tests --- cloudinit/reporting/handlers.py | 2 +- tests/unittests/test_reporting.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'cloudinit/reporting/handlers.py') diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 1343311f..172679cc 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -35,7 +35,7 @@ class LogHandler(ReportingHandler): else: input_level = level try: - level = gettattr(logging, level.upper()) + level = getattr(logging, level.upper()) except: LOG.warn("invalid level '%s', using WARN", input_level) level = logging.WARN diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 1a4ee8c4..66d4e87e 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -137,14 +137,14 @@ class TestLogHandler(TestCase): def test_single_log_message_at_info_published(self, getLogger): event = reporting.ReportingEvent('type', 'name', 'description') reporting.handlers.LogHandler().publish_event(event) - self.assertEqual(1, getLogger.return_value.info.call_count) + self.assertEqual(1, getLogger.return_value.log.call_count) @mock.patch.object(reporting.handlers.logging, 'getLogger') def test_log_message_uses_event_as_string(self, getLogger): event = reporting.ReportingEvent('type', 'name', 'description') - reporting.handlers.LogHandler().publish_event(event) + reporting.handlers.LogHandler(level="INFO").publish_event(event) self.assertIn(event.as_string(), - getLogger.return_value.info.call_args[0][0]) + getLogger.return_value.log.call_args[0][1]) class TestDefaultRegisteredHandler(TestCase): -- cgit v1.2.3 From a9c1e3f747ae69401ebfca9ae64eec1c6d20ebe7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 10 Aug 2015 11:17:28 -0400 Subject: reporting: remove unused variable, actually print in PrintHandler --- cloudinit/reporting/__init__.py | 1 - cloudinit/reporting/handlers.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/reporting/handlers.py') diff --git a/cloudinit/reporting/__init__.py b/cloudinit/reporting/__init__.py index e23fab32..502af95c 100644 --- a/cloudinit/reporting/__init__.py +++ b/cloudinit/reporting/__init__.py @@ -83,7 +83,6 @@ def update_configuration(config): instantiated_handler_registry.unregister_item( handler_name, force=True) continue - registered = instantiated_handler_registry.registered_items handler_config = handler_config.copy() cls = available_handlers.registered_items[handler_config.pop('type')] instantiated_handler_registry.unregister_item(handler_name) diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 172679cc..5ed3cb84 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -23,6 +23,7 @@ class ReportingHandler(object): @abc.abstractmethod def publish_event(self, event): """Publish an event to the ``INFO`` log level.""" + print(event.as_string()) class LogHandler(ReportingHandler): -- cgit v1.2.3 From 8bcccd07d1dbde74126e81967388d2e5a90fcfa7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 31 Aug 2015 14:10:54 -0400 Subject: handlers: docstring fixups, and print actually do something --- cloudinit/reporting/handlers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'cloudinit/reporting/handlers.py') diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 5ed3cb84..140a98c5 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -22,12 +22,11 @@ class ReportingHandler(object): @abc.abstractmethod def publish_event(self, event): - """Publish an event to the ``INFO`` log level.""" - print(event.as_string()) + """Publish an event.""" class LogHandler(ReportingHandler): - """Publishes events to the cloud-init log at the ``INFO`` log level.""" + """Publishes events to the cloud-init log at the ``DEBUG`` log level.""" def __init__(self, level="DEBUG"): super(LogHandler, self).__init__() @@ -43,15 +42,16 @@ class LogHandler(ReportingHandler): self.level = level def publish_event(self, event): - """Publish an event to the ``INFO`` log level.""" logger = logging.getLogger( '.'.join(['cloudinit', 'reporting', event.event_type, event.name])) logger.log(self.level, event.as_string()) class PrintHandler(ReportingHandler): + """Print the event as a string.""" + def publish_event(self, event): - """Publish an event to the ``INFO`` log level.""" + print(event.as_string()) class WebHookHandler(ReportingHandler): -- cgit v1.2.3 From 3f2dddae6e8d5148bcf89c2b4e27975d1da77aea Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 31 Aug 2015 14:11:47 -0400 Subject: handlers: drop unused import this import was left over from before we moved oauthlib into url_helper --- cloudinit/reporting/handlers.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit/reporting/handlers.py') diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 140a98c5..ba480da0 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -1,7 +1,6 @@ # vi: ts=4 expandtab import abc -import oauthlib.oauth1 as oauth1 import six from ..registry import DictRegistry -- cgit v1.2.3 From 4558922ac6d8ae129b1f47e124c6b08008e7548f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 17 Sep 2015 15:56:51 -0400 Subject: webhook: report with json data the handler was passing a dictionary to readurl which was then passing that on to requests.request as 'data'. the requests library would urlencode that, but we want the json data posted instead. LP: #1496960 --- cloudinit/reporting/handlers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit/reporting/handlers.py') diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index ba480da0..3212d173 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -1,6 +1,7 @@ # vi: ts=4 expandtab import abc +import json import six from ..registry import DictRegistry @@ -77,7 +78,7 @@ class WebHookHandler(ReportingHandler): readurl = url_helper.readurl try: return readurl( - self.endpoint, data=event.as_dict(), + self.endpoint, data=json.dumps(event.as_dict()), timeout=self.timeout, retries=self.retries, ssl_details=self.ssl_details) except: -- cgit v1.2.3