From 6010d3c8b903d7dae8b0ff11ec45c6f78ea50cc8 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Mon, 31 Aug 2015 12:28:09 -0400
Subject: readurl: if headers are provided still provide base headers

we want cloud-init user agent to be present even if the user
provided some headers. In the event that they provided User-Agent,
this will respect their wishes.
---
 cloudinit/url_helper.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

(limited to 'cloudinit')

diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index ce6b5444..a93847ce 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -211,10 +211,14 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
     manual_tries = 1
     if retries:
         manual_tries = max(int(retries) + 1, 1)
-    if not headers:
-        headers = {
-            'User-Agent': 'Cloud-Init/%s' % (version.version_string()),
-        }
+
+    def_headers = {
+        'User-Agent': 'Cloud-Init/%s' % (version.version_string()),
+    }
+    if headers:
+        def_headers.update(headers)
+    headers = def_headers
+
     if not headers_cb:
         def _cb(url):
             return headers
-- 
cgit v1.2.3


From 50bcb0f77d29a76a03946c6da13b15be25257402 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Mon, 31 Aug 2015 13:33:30 -0400
Subject: split 'events' portion of reporting into separate file

this just separates events from other things that could conceivably
be reported.
---
 cloudinit/cloud.py                |   4 +-
 cloudinit/reporting/__init__.py   | 203 +-----------------------------------
 cloudinit/reporting/events.py     | 210 ++++++++++++++++++++++++++++++++++++++
 cloudinit/sources/__init__.py     |   4 +-
 cloudinit/stages.py               |  14 +--
 tests/unittests/test_reporting.py | 121 +++++++++++-----------
 6 files changed, 285 insertions(+), 271 deletions(-)
 create mode 100644 cloudinit/reporting/events.py

(limited to 'cloudinit')

diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py
index edee3887..3e6be203 100644
--- a/cloudinit/cloud.py
+++ b/cloudinit/cloud.py
@@ -24,7 +24,7 @@ import copy
 import os
 
 from cloudinit import log as logging
-from cloudinit import reporting
+from cloudinit.reporting import events
 
 LOG = logging.getLogger(__name__)
 
@@ -48,7 +48,7 @@ class Cloud(object):
         self._cfg = cfg
         self._runners = runners
         if reporter is None:
-            reporter = reporting.ReportEventStack(
+            reporter = events.ReportEventStack(
                 name="unnamed-cloud-reporter",
                 description="unnamed-cloud-reporter",
                 reporting_enabled=False)
diff --git a/cloudinit/reporting/__init__.py b/cloudinit/reporting/__init__.py
index 502af95c..6b41ae61 100644
--- a/cloudinit/reporting/__init__.py
+++ b/cloudinit/reporting/__init__.py
@@ -1,7 +1,6 @@
 # Copyright 2015 Canonical Ltd.
 # This file is part of cloud-init.  See LICENCE file for license information.
 #
-# vi: ts=4 expandtab
 """
 cloud-init reporting framework
 
@@ -10,66 +9,13 @@ report events in a structured manner.
 """
 
 from ..registry import DictRegistry
-from ..reporting.handlers import available_handlers
-
-
-FINISH_EVENT_TYPE = 'finish'
-START_EVENT_TYPE = 'start'
+from .handlers import available_handlers
 
 DEFAULT_CONFIG = {
     'logging': {'type': 'log'},
 }
 
 
-class _nameset(set):
-    def __getattr__(self, name):
-        if name in self:
-            return name
-        raise AttributeError("%s not a valid value" % name)
-
-
-status = _nameset(("SUCCESS", "WARN", "FAIL"))
-
-
-class ReportingEvent(object):
-    """Encapsulation of event formatting."""
-
-    def __init__(self, event_type, name, description):
-        self.event_type = event_type
-        self.name = name
-        self.description = description
-
-    def as_string(self):
-        """The event represented as a string."""
-        return '{0}: {1}: {2}'.format(
-            self.event_type, self.name, self.description)
-
-    def as_dict(self):
-        """The event represented as a dictionary."""
-        return {'name': self.name, 'description': self.description,
-                'event_type': self.event_type}
-
-
-class FinishReportingEvent(ReportingEvent):
-
-    def __init__(self, name, description, result=status.SUCCESS):
-        super(FinishReportingEvent, self).__init__(
-            FINISH_EVENT_TYPE, name, description)
-        self.result = result
-        if result not in status:
-            raise ValueError("Invalid result: %s" % result)
-
-    def as_string(self):
-        return '{0}: {1}: {2}: {3}'.format(
-            self.event_type, self.name, self.result, self.description)
-
-    def as_dict(self):
-        """The event represented as json friendly."""
-        data = super(FinishReportingEvent, self).as_dict()
-        data['result'] = self.result
-        return data
-
-
 def update_configuration(config):
     """Update the instanciated_handler_registry.
 
@@ -90,150 +36,7 @@ def update_configuration(config):
         instantiated_handler_registry.register_item(handler_name, instance)
 
 
-def report_event(event):
-    """Report an event to all registered event handlers.
-
-    This should generally be called via one of the other functions in
-    the reporting module.
-
-    :param event_type:
-        The type of the event; this should be a constant from the
-        reporting module.
-    """
-    for _, handler in instantiated_handler_registry.registered_items.items():
-        handler.publish_event(event)
-
-
-def report_finish_event(event_name, event_description,
-                        result=status.SUCCESS):
-    """Report a "finish" event.
-
-    See :py:func:`.report_event` for parameter details.
-    """
-    event = FinishReportingEvent(event_name, event_description, result)
-    return report_event(event)
-
-
-def report_start_event(event_name, event_description):
-    """Report a "start" event.
-
-    :param event_name:
-        The name of the event; this should be a topic which events would
-        share (e.g. it will be the same for start and finish events).
-
-    :param event_description:
-        A human-readable description of the event that has occurred.
-    """
-    event = ReportingEvent(START_EVENT_TYPE, event_name, event_description)
-    return report_event(event)
-
-
-class ReportEventStack(object):
-    """Context Manager for using :py:func:`report_event`
-
-    This enables calling :py:func:`report_start_event` and
-    :py:func:`report_finish_event` through a context manager.
-
-    :param name:
-        the name of the event
-
-    :param description:
-        the event's description, passed on to :py:func:`report_start_event`
-
-    :param message:
-        the description to use for the finish event. defaults to
-        :param:description.
-
-    :param parent:
-    :type parent: :py:class:ReportEventStack or None
-        The parent of this event.  The parent is populated with
-        results of all its children.  The name used in reporting
-        is <parent.name>/<name>
-
-    :param reporting_enabled:
-        Indicates if reporting events should be generated.
-        If not provided, defaults to the parent's value, or True if no parent
-        is provided.
-
-    :param result_on_exception:
-        The result value to set if an exception is caught. default
-        value is FAIL.
-    """
-    def __init__(self, name, description, message=None, parent=None,
-                 reporting_enabled=None, result_on_exception=status.FAIL):
-        self.parent = parent
-        self.name = name
-        self.description = description
-        self.message = message
-        self.result_on_exception = result_on_exception
-        self.result = status.SUCCESS
-
-        # use parents reporting value if not provided
-        if reporting_enabled is None:
-            if parent:
-                reporting_enabled = parent.reporting_enabled
-            else:
-                reporting_enabled = True
-        self.reporting_enabled = reporting_enabled
-
-        if parent:
-            self.fullname = '/'.join((parent.fullname, name,))
-        else:
-            self.fullname = self.name
-        self.children = {}
-
-    def __repr__(self):
-        return ("ReportEventStack(%s, %s, reporting_enabled=%s)" %
-                (self.name, self.description, self.reporting_enabled))
-
-    def __enter__(self):
-        self.result = status.SUCCESS
-        if self.reporting_enabled:
-            report_start_event(self.fullname, self.description)
-        if self.parent:
-            self.parent.children[self.name] = (None, None)
-        return self
-
-    def _childrens_finish_info(self):
-        for cand_result in (status.FAIL, status.WARN):
-            for name, (value, msg) in self.children.items():
-                if value == cand_result:
-                    return (value, self.message)
-        return (self.result, self.message)
-
-    @property
-    def result(self):
-        return self._result
-
-    @result.setter
-    def result(self, value):
-        if value not in status:
-            raise ValueError("'%s' not a valid result" % value)
-        self._result = value
-
-    @property
-    def message(self):
-        if self._message is not None:
-            return self._message
-        return self.description
-
-    @message.setter
-    def message(self, value):
-        self._message = value
-
-    def _finish_info(self, exc):
-        # return tuple of description, and value
-        if exc:
-            return (self.result_on_exception, self.message)
-        return self._childrens_finish_info()
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        (result, msg) = self._finish_info(exc_value)
-        if self.parent:
-            self.parent.children[self.name] = (result, msg)
-        if self.reporting_enabled:
-            report_finish_event(self.fullname, msg, result)
-
-
 instantiated_handler_registry = DictRegistry()
 update_configuration(DEFAULT_CONFIG)
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/reporting/events.py b/cloudinit/reporting/events.py
new file mode 100644
index 00000000..e35e41dd
--- /dev/null
+++ b/cloudinit/reporting/events.py
@@ -0,0 +1,210 @@
+# Copyright 2015 Canonical Ltd.
+# This file is part of cloud-init.  See LICENCE file for license information.
+#
+"""
+cloud-init events
+
+Report events in a structured manner.
+The events here are most likely used via reporting.
+"""
+
+from . import instantiated_handler_registry
+
+FINISH_EVENT_TYPE = 'finish'
+START_EVENT_TYPE = 'start'
+
+
+class _nameset(set):
+    def __getattr__(self, name):
+        if name in self:
+            return name
+        raise AttributeError("%s not a valid value" % name)
+
+
+status = _nameset(("SUCCESS", "WARN", "FAIL"))
+
+
+class ReportingEvent(object):
+    """Encapsulation of event formatting."""
+
+    def __init__(self, event_type, name, description):
+        self.event_type = event_type
+        self.name = name
+        self.description = description
+
+    def as_string(self):
+        """The event represented as a string."""
+        return '{0}: {1}: {2}'.format(
+            self.event_type, self.name, self.description)
+
+    def as_dict(self):
+        """The event represented as a dictionary."""
+        return {'name': self.name, 'description': self.description,
+                'event_type': self.event_type}
+
+
+class FinishReportingEvent(ReportingEvent):
+
+    def __init__(self, name, description, result=status.SUCCESS):
+        super(FinishReportingEvent, self).__init__(
+            FINISH_EVENT_TYPE, name, description)
+        self.result = result
+        if result not in status:
+            raise ValueError("Invalid result: %s" % result)
+
+    def as_string(self):
+        return '{0}: {1}: {2}: {3}'.format(
+            self.event_type, self.name, self.result, self.description)
+
+    def as_dict(self):
+        """The event represented as json friendly."""
+        data = super(FinishReportingEvent, self).as_dict()
+        data['result'] = self.result
+        return data
+
+
+def report_event(event):
+    """Report an event to all registered event handlers.
+
+    This should generally be called via one of the other functions in
+    the reporting module.
+
+    :param event_type:
+        The type of the event; this should be a constant from the
+        reporting module.
+    """
+    for _, handler in instantiated_handler_registry.registered_items.items():
+        handler.publish_event(event)
+
+
+def report_finish_event(event_name, event_description,
+                        result=status.SUCCESS):
+    """Report a "finish" event.
+
+    See :py:func:`.report_event` for parameter details.
+    """
+    event = FinishReportingEvent(event_name, event_description, result)
+    return report_event(event)
+
+
+def report_start_event(event_name, event_description):
+    """Report a "start" event.
+
+    :param event_name:
+        The name of the event; this should be a topic which events would
+        share (e.g. it will be the same for start and finish events).
+
+    :param event_description:
+        A human-readable description of the event that has occurred.
+    """
+    event = ReportingEvent(START_EVENT_TYPE, event_name, event_description)
+    return report_event(event)
+
+
+class ReportEventStack(object):
+    """Context Manager for using :py:func:`report_event`
+
+    This enables calling :py:func:`report_start_event` and
+    :py:func:`report_finish_event` through a context manager.
+
+    :param name:
+        the name of the event
+
+    :param description:
+        the event's description, passed on to :py:func:`report_start_event`
+
+    :param message:
+        the description to use for the finish event. defaults to
+        :param:description.
+
+    :param parent:
+    :type parent: :py:class:ReportEventStack or None
+        The parent of this event.  The parent is populated with
+        results of all its children.  The name used in reporting
+        is <parent.name>/<name>
+
+    :param reporting_enabled:
+        Indicates if reporting events should be generated.
+        If not provided, defaults to the parent's value, or True if no parent
+        is provided.
+
+    :param result_on_exception:
+        The result value to set if an exception is caught. default
+        value is FAIL.
+    """
+    def __init__(self, name, description, message=None, parent=None,
+                 reporting_enabled=None, result_on_exception=status.FAIL):
+        self.parent = parent
+        self.name = name
+        self.description = description
+        self.message = message
+        self.result_on_exception = result_on_exception
+        self.result = status.SUCCESS
+
+        # use parents reporting value if not provided
+        if reporting_enabled is None:
+            if parent:
+                reporting_enabled = parent.reporting_enabled
+            else:
+                reporting_enabled = True
+        self.reporting_enabled = reporting_enabled
+
+        if parent:
+            self.fullname = '/'.join((parent.fullname, name,))
+        else:
+            self.fullname = self.name
+        self.children = {}
+
+    def __repr__(self):
+        return ("ReportEventStack(%s, %s, reporting_enabled=%s)" %
+                (self.name, self.description, self.reporting_enabled))
+
+    def __enter__(self):
+        self.result = status.SUCCESS
+        if self.reporting_enabled:
+            report_start_event(self.fullname, self.description)
+        if self.parent:
+            self.parent.children[self.name] = (None, None)
+        return self
+
+    def _childrens_finish_info(self):
+        for cand_result in (status.FAIL, status.WARN):
+            for name, (value, msg) in self.children.items():
+                if value == cand_result:
+                    return (value, self.message)
+        return (self.result, self.message)
+
+    @property
+    def result(self):
+        return self._result
+
+    @result.setter
+    def result(self, value):
+        if value not in status:
+            raise ValueError("'%s' not a valid result" % value)
+        self._result = value
+
+    @property
+    def message(self):
+        if self._message is not None:
+            return self._message
+        return self.description
+
+    @message.setter
+    def message(self, value):
+        self._message = value
+
+    def _finish_info(self, exc):
+        # return tuple of description, and value
+        if exc:
+            return (self.result_on_exception, self.message)
+        return self._childrens_finish_info()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        (result, msg) = self._finish_info(exc_value)
+        if self.parent:
+            self.parent.children[self.name] = (result, msg)
+        if self.reporting_enabled:
+            report_finish_event(self.fullname, msg, result)
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 838cd198..d3cfa560 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -27,12 +27,12 @@ import six
 
 from cloudinit import importer
 from cloudinit import log as logging
-from cloudinit import reporting
 from cloudinit import type_utils
 from cloudinit import user_data as ud
 from cloudinit import util
 
 from cloudinit.filters import launch_index
+from cloudinit.reporting import events
 
 DEP_FILESYSTEM = "FILESYSTEM"
 DEP_NETWORK = "NETWORK"
@@ -254,7 +254,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter):
     LOG.debug("Searching for %s data source in: %s", mode, ds_names)
 
     for name, cls in zip(ds_names, ds_list):
-        myrep = reporting.ReportEventStack(
+        myrep = events.ReportEventStack(
             name="search-%s" % name.replace("DataSource", ""),
             description="searching for %s data from %s" % (mode, name),
             message="no %s data found from %s" % (mode, name),
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index d300709d..9f192c8d 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -46,7 +46,7 @@ from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import type_utils
 from cloudinit import util
-from cloudinit import reporting
+from cloudinit.reporting import events
 
 LOG = logging.getLogger(__name__)
 
@@ -67,7 +67,7 @@ class Init(object):
         self.datasource = NULL_DATA_SOURCE
 
         if reporter is None:
-            reporter = reporting.ReportEventStack(
+            reporter = events.ReportEventStack(
                 name="init-reporter", description="init-desc",
                 reporting_enabled=False)
         self.reporter = reporter
@@ -242,7 +242,7 @@ class Init(object):
         if self.datasource is not NULL_DATA_SOURCE:
             return self.datasource
 
-        with reporting.ReportEventStack(
+        with events.ReportEventStack(
                 name="check-cache",
                 description="attempting to read from cache",
                 parent=self.reporter) as myrep:
@@ -509,11 +509,11 @@ class Init(object):
     def consume_data(self, frequency=PER_INSTANCE):
         # Consume the userdata first, because we need want to let the part
         # handlers run first (for merging stuff)
-        with reporting.ReportEventStack(
+        with events.ReportEventStack(
             "consume-user-data", "reading and applying user-data",
             parent=self.reporter):
                 self._consume_userdata(frequency)
-        with reporting.ReportEventStack(
+        with events.ReportEventStack(
             "consume-vendor-data", "reading and applying vendor-data",
             parent=self.reporter):
                 self._consume_vendordata(frequency)
@@ -595,7 +595,7 @@ class Modules(object):
         # Created on first use
         self._cached_cfg = None
         if reporter is None:
-            reporter = reporting.ReportEventStack(
+            reporter = events.ReportEventStack(
                 name="module-reporter", description="module-desc",
                 reporting_enabled=False)
         self.reporter = reporter
@@ -710,7 +710,7 @@ class Modules(object):
                 run_name = "config-%s" % (name)
 
                 desc = "running %s with frequency %s" % (run_name, freq)
-                myrep = reporting.ReportEventStack(
+                myrep = events.ReportEventStack(
                     name=run_name, description=desc, parent=self.reporter)
 
                 with myrep:
diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py
index 66d4e87e..bb67ef73 100644
--- a/tests/unittests/test_reporting.py
+++ b/tests/unittests/test_reporting.py
@@ -5,6 +5,7 @@
 
 from cloudinit import reporting
 from cloudinit.reporting import handlers
+from cloudinit.reporting import events
 
 from .helpers import (mock, TestCase)
 
@@ -16,12 +17,12 @@ def _fake_registry():
 
 class TestReportStartEvent(TestCase):
 
-    @mock.patch('cloudinit.reporting.instantiated_handler_registry',
+    @mock.patch('cloudinit.reporting.events.instantiated_handler_registry',
                 new_callable=_fake_registry)
     def test_report_start_event_passes_something_with_as_string_to_handlers(
             self, instantiated_handler_registry):
         event_name, event_description = 'my_test_event', 'my description'
-        reporting.report_start_event(event_name, event_description)
+        events.report_start_event(event_name, event_description)
         expected_string_representation = ': '.join(
             ['start', event_name, event_description])
         for _, handler in (
@@ -33,9 +34,9 @@ class TestReportStartEvent(TestCase):
 
 class TestReportFinishEvent(TestCase):
 
-    def _report_finish_event(self, result=reporting.status.SUCCESS):
+    def _report_finish_event(self, result=events.status.SUCCESS):
         event_name, event_description = 'my_test_event', 'my description'
-        reporting.report_finish_event(
+        events.report_finish_event(
             event_name, event_description, result=result)
         return event_name, event_description
 
@@ -46,39 +47,39 @@ class TestReportFinishEvent(TestCase):
             event = handler.publish_event.call_args[0][0]
             self.assertEqual(expected_as_string, event.as_string())
 
-    @mock.patch('cloudinit.reporting.instantiated_handler_registry',
+    @mock.patch('cloudinit.reporting.events.instantiated_handler_registry',
                 new_callable=_fake_registry)
     def test_report_finish_event_passes_something_with_as_string_to_handlers(
             self, instantiated_handler_registry):
         event_name, event_description = self._report_finish_event()
         expected_string_representation = ': '.join(
-            ['finish', event_name, reporting.status.SUCCESS,
+            ['finish', event_name, events.status.SUCCESS,
              event_description])
         self.assertHandlersPassedObjectWithAsString(
             instantiated_handler_registry.registered_items,
             expected_string_representation)
 
-    @mock.patch('cloudinit.reporting.instantiated_handler_registry',
+    @mock.patch('cloudinit.reporting.events.instantiated_handler_registry',
                 new_callable=_fake_registry)
     def test_reporting_successful_finish_has_sensible_string_repr(
             self, instantiated_handler_registry):
         event_name, event_description = self._report_finish_event(
-            result=reporting.status.SUCCESS)
+            result=events.status.SUCCESS)
         expected_string_representation = ': '.join(
-            ['finish', event_name, reporting.status.SUCCESS,
+            ['finish', event_name, events.status.SUCCESS,
              event_description])
         self.assertHandlersPassedObjectWithAsString(
             instantiated_handler_registry.registered_items,
             expected_string_representation)
 
-    @mock.patch('cloudinit.reporting.instantiated_handler_registry',
+    @mock.patch('cloudinit.reporting.events.instantiated_handler_registry',
                 new_callable=_fake_registry)
     def test_reporting_unsuccessful_finish_has_sensible_string_repr(
             self, instantiated_handler_registry):
         event_name, event_description = self._report_finish_event(
-            result=reporting.status.FAIL)
+            result=events.status.FAIL)
         expected_string_representation = ': '.join(
-            ['finish', event_name, reporting.status.FAIL, event_description])
+            ['finish', event_name, events.status.FAIL, event_description])
         self.assertHandlersPassedObjectWithAsString(
             instantiated_handler_registry.registered_items,
             expected_string_representation)
@@ -91,14 +92,14 @@ class TestReportingEvent(TestCase):
 
     def test_as_string(self):
         event_type, name, description = 'test_type', 'test_name', 'test_desc'
-        event = reporting.ReportingEvent(event_type, name, description)
+        event = events.ReportingEvent(event_type, name, description)
         expected_string_representation = ': '.join(
             [event_type, name, description])
         self.assertEqual(expected_string_representation, event.as_string())
 
     def test_as_dict(self):
         event_type, name, desc = 'test_type', 'test_name', 'test_desc'
-        event = reporting.ReportingEvent(event_type, name, desc)
+        event = events.ReportingEvent(event_type, name, desc)
         self.assertEqual(
             {'event_type': event_type, 'name': name, 'description': desc},
             event.as_dict())
@@ -106,9 +107,9 @@ class TestReportingEvent(TestCase):
 
 class TestFinishReportingEvent(TestCase):
     def test_as_has_result(self):
-        result = reporting.status.SUCCESS
+        result = events.status.SUCCESS
         name, desc = 'test_name', 'test_desc'
-        event = reporting.FinishReportingEvent(name, desc, result)
+        event = events.FinishReportingEvent(name, desc, result)
         ret = event.as_dict()
         self.assertTrue('result' in ret)
         self.assertEqual(ret['result'], result)
@@ -126,7 +127,7 @@ class TestLogHandler(TestCase):
     @mock.patch.object(reporting.handlers.logging, 'getLogger')
     def test_appropriate_logger_used(self, getLogger):
         event_type, event_name = 'test_type', 'test_name'
-        event = reporting.ReportingEvent(event_type, event_name, 'description')
+        event = events.ReportingEvent(event_type, event_name, 'description')
         reporting.handlers.LogHandler().publish_event(event)
         self.assertEqual(
             [mock.call(
@@ -135,13 +136,13 @@ class TestLogHandler(TestCase):
 
     @mock.patch.object(reporting.handlers.logging, 'getLogger')
     def test_single_log_message_at_info_published(self, getLogger):
-        event = reporting.ReportingEvent('type', 'name', 'description')
+        event = events.ReportingEvent('type', 'name', 'description')
         reporting.handlers.LogHandler().publish_event(event)
         self.assertEqual(1, getLogger.return_value.log.call_count)
 
     @mock.patch.object(reporting.handlers.logging, 'getLogger')
     def test_log_message_uses_event_as_string(self, getLogger):
-        event = reporting.ReportingEvent('type', 'name', 'description')
+        event = events.ReportingEvent('type', 'name', 'description')
         reporting.handlers.LogHandler(level="INFO").publish_event(event)
         self.assertIn(event.as_string(),
                       getLogger.return_value.log.call_args[0][1])
@@ -232,49 +233,49 @@ class TestReportingConfiguration(TestCase):
 
 
 class TestReportingEventStack(TestCase):
-    @mock.patch('cloudinit.reporting.report_finish_event')
-    @mock.patch('cloudinit.reporting.report_start_event')
+    @mock.patch('cloudinit.reporting.events.report_finish_event')
+    @mock.patch('cloudinit.reporting.events.report_start_event')
     def test_start_and_finish_success(self, report_start, report_finish):
-        with reporting.ReportEventStack(name="myname", description="mydesc"):
+        with events.ReportEventStack(name="myname", description="mydesc"):
             pass
         self.assertEqual(
             [mock.call('myname', 'mydesc')], report_start.call_args_list)
         self.assertEqual(
-            [mock.call('myname', 'mydesc', reporting.status.SUCCESS)],
+            [mock.call('myname', 'mydesc', events.status.SUCCESS)],
             report_finish.call_args_list)
 
-    @mock.patch('cloudinit.reporting.report_finish_event')
-    @mock.patch('cloudinit.reporting.report_start_event')
+    @mock.patch('cloudinit.reporting.events.report_finish_event')
+    @mock.patch('cloudinit.reporting.events.report_start_event')
     def test_finish_exception_defaults_fail(self, report_start, report_finish):
         name = "myname"
         desc = "mydesc"
         try:
-            with reporting.ReportEventStack(name, description=desc):
+            with events.ReportEventStack(name, description=desc):
                 raise ValueError("This didnt work")
         except ValueError:
             pass
         self.assertEqual([mock.call(name, desc)], report_start.call_args_list)
         self.assertEqual(
-            [mock.call(name, desc, reporting.status.FAIL)],
+            [mock.call(name, desc, events.status.FAIL)],
             report_finish.call_args_list)
 
-    @mock.patch('cloudinit.reporting.report_finish_event')
-    @mock.patch('cloudinit.reporting.report_start_event')
+    @mock.patch('cloudinit.reporting.events.report_finish_event')
+    @mock.patch('cloudinit.reporting.events.report_start_event')
     def test_result_on_exception_used(self, report_start, report_finish):
         name = "myname"
         desc = "mydesc"
         try:
-            with reporting.ReportEventStack(
-                    name, desc, result_on_exception=reporting.status.WARN):
+            with events.ReportEventStack(
+                    name, desc, result_on_exception=events.status.WARN):
                 raise ValueError("This didnt work")
         except ValueError:
             pass
         self.assertEqual([mock.call(name, desc)], report_start.call_args_list)
         self.assertEqual(
-            [mock.call(name, desc, reporting.status.WARN)],
+            [mock.call(name, desc, events.status.WARN)],
             report_finish.call_args_list)
 
-    @mock.patch('cloudinit.reporting.report_start_event')
+    @mock.patch('cloudinit.reporting.events.report_start_event')
     def test_child_fullname_respects_parent(self, report_start):
         parent_name = "topname"
         c1_name = "c1name"
@@ -282,59 +283,59 @@ class TestReportingEventStack(TestCase):
         c2_expected_fullname = '/'.join([parent_name, c1_name, c2_name])
         c1_expected_fullname = '/'.join([parent_name, c1_name])
 
-        parent = reporting.ReportEventStack(parent_name, "topdesc")
-        c1 = reporting.ReportEventStack(c1_name, "c1desc", parent=parent)
-        c2 = reporting.ReportEventStack(c2_name, "c2desc", parent=c1)
+        parent = events.ReportEventStack(parent_name, "topdesc")
+        c1 = events.ReportEventStack(c1_name, "c1desc", parent=parent)
+        c2 = events.ReportEventStack(c2_name, "c2desc", parent=c1)
         with c1:
             report_start.assert_called_with(c1_expected_fullname, "c1desc")
             with c2:
                 report_start.assert_called_with(c2_expected_fullname, "c2desc")
 
-    @mock.patch('cloudinit.reporting.report_finish_event')
-    @mock.patch('cloudinit.reporting.report_start_event')
+    @mock.patch('cloudinit.reporting.events.report_finish_event')
+    @mock.patch('cloudinit.reporting.events.report_start_event')
     def test_child_result_bubbles_up(self, report_start, report_finish):
-        parent = reporting.ReportEventStack("topname", "topdesc")
-        child = reporting.ReportEventStack("c_name", "c_desc", parent=parent)
+        parent = events.ReportEventStack("topname", "topdesc")
+        child = events.ReportEventStack("c_name", "c_desc", parent=parent)
         with parent:
             with child:
-                child.result = reporting.status.WARN
+                child.result = events.status.WARN
 
         report_finish.assert_called_with(
-            "topname", "topdesc", reporting.status.WARN)
+            "topname", "topdesc", events.status.WARN)
 
-    @mock.patch('cloudinit.reporting.report_finish_event')
+    @mock.patch('cloudinit.reporting.events.report_finish_event')
     def test_message_used_in_finish(self, report_finish):
-        with reporting.ReportEventStack("myname", "mydesc",
-                                        message="mymessage"):
+        with events.ReportEventStack("myname", "mydesc",
+                                     message="mymessage"):
             pass
         self.assertEqual(
-            [mock.call("myname", "mymessage", reporting.status.SUCCESS)],
+            [mock.call("myname", "mymessage", events.status.SUCCESS)],
             report_finish.call_args_list)
 
-    @mock.patch('cloudinit.reporting.report_finish_event')
+    @mock.patch('cloudinit.reporting.events.report_finish_event')
     def test_message_updatable(self, report_finish):
-        with reporting.ReportEventStack("myname", "mydesc") as c:
+        with events.ReportEventStack("myname", "mydesc") as c:
             c.message = "all good"
         self.assertEqual(
-            [mock.call("myname", "all good", reporting.status.SUCCESS)],
+            [mock.call("myname", "all good", events.status.SUCCESS)],
             report_finish.call_args_list)
 
-    @mock.patch('cloudinit.reporting.report_start_event')
-    @mock.patch('cloudinit.reporting.report_finish_event')
+    @mock.patch('cloudinit.reporting.events.report_start_event')
+    @mock.patch('cloudinit.reporting.events.report_finish_event')
     def test_reporting_disabled_does_not_report_events(
             self, report_start, report_finish):
-        with reporting.ReportEventStack("a", "b", reporting_enabled=False):
+        with events.ReportEventStack("a", "b", reporting_enabled=False):
             pass
         self.assertEqual(report_start.call_count, 0)
         self.assertEqual(report_finish.call_count, 0)
 
-    @mock.patch('cloudinit.reporting.report_start_event')
-    @mock.patch('cloudinit.reporting.report_finish_event')
+    @mock.patch('cloudinit.reporting.events.report_start_event')
+    @mock.patch('cloudinit.reporting.events.report_finish_event')
     def test_reporting_child_default_to_parent(
             self, report_start, report_finish):
-        parent = reporting.ReportEventStack(
+        parent = events.ReportEventStack(
             "pname", "pdesc", reporting_enabled=False)
-        child = reporting.ReportEventStack("cname", "cdesc", parent=parent)
+        child = events.ReportEventStack("cname", "cdesc", parent=parent)
         with parent:
             with child:
                 pass
@@ -343,17 +344,17 @@ class TestReportingEventStack(TestCase):
         self.assertEqual(report_finish.call_count, 0)
 
     def test_reporting_event_has_sane_repr(self):
-        myrep = reporting.ReportEventStack("fooname", "foodesc",
-                                           reporting_enabled=True).__repr__()
+        myrep = events.ReportEventStack("fooname", "foodesc",
+                                        reporting_enabled=True).__repr__()
         self.assertIn("fooname", myrep)
         self.assertIn("foodesc", myrep)
         self.assertIn("True", myrep)
 
     def test_set_invalid_result_raises_value_error(self):
-        f = reporting.ReportEventStack("myname", "mydesc")
+        f = events.ReportEventStack("myname", "mydesc")
         self.assertRaises(ValueError, setattr, f, "result", "BOGUS")
 
 
 class TestStatusAccess(TestCase):
     def test_invalid_status_access_raises_value_error(self):
-        self.assertRaises(AttributeError, getattr, reporting.status, "BOGUS")
+        self.assertRaises(AttributeError, getattr, events.status, "BOGUS")
-- 
cgit v1.2.3


From 7820a43baf94e11ce458476a442edd726a406aba Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Mon, 31 Aug 2015 13:57:05 -0400
Subject: events: add timestamp and origin, support file posting

This adds 'timestamp' and 'origin' to events.
The timestamp is simply that, a floating point timestamp of when
the event occurred.

The origin indicates the source / reporter of this.  It is useful
to have a single endpoint with multiple different things reporting
to it.  For example, MAAS will configure cloud-init and curtin
to report to the same endpoint and then it can differenciate who
made the post.  Admittedly, they could use multiple endpoints, but
this this seems sane.

Also, add support for posting files at the close of an event.
This is utilized in curtin to post a log file when the install is
done.  files are posted on success or fail of the event.
---
 cloudinit/reporting/events.py     | 58 +++++++++++++++++++++++++++++++--------
 tests/unittests/test_reporting.py | 15 ++++++----
 2 files changed, 56 insertions(+), 17 deletions(-)

(limited to 'cloudinit')

diff --git a/cloudinit/reporting/events.py b/cloudinit/reporting/events.py
index e35e41dd..2f767f64 100644
--- a/cloudinit/reporting/events.py
+++ b/cloudinit/reporting/events.py
@@ -2,17 +2,22 @@
 # This file is part of cloud-init.  See LICENCE file for license information.
 #
 """
-cloud-init events
+events for reporting.
 
-Report events in a structured manner.
-The events here are most likely used via reporting.
+The events here are designed to be used with reporting.
+They can be published to registered handlers with report_event.
 """
+import base64
+import os.path
+import time
 
 from . import instantiated_handler_registry
 
 FINISH_EVENT_TYPE = 'finish'
 START_EVENT_TYPE = 'start'
 
+DEFAULT_EVENT_ORIGIN = 'cloudinit'
+
 
 class _nameset(set):
     def __getattr__(self, name):
@@ -27,10 +32,13 @@ status = _nameset(("SUCCESS", "WARN", "FAIL"))
 class ReportingEvent(object):
     """Encapsulation of event formatting."""
 
-    def __init__(self, event_type, name, description):
+    def __init__(self, event_type, name, description,
+                 origin=DEFAULT_EVENT_ORIGIN, timestamp=time.time()):
         self.event_type = event_type
         self.name = name
         self.description = description
+        self.origin = origin
+        self.timestamp = timestamp
 
     def as_string(self):
         """The event represented as a string."""
@@ -40,15 +48,20 @@ class ReportingEvent(object):
     def as_dict(self):
         """The event represented as a dictionary."""
         return {'name': self.name, 'description': self.description,
-                'event_type': self.event_type}
+                'event_type': self.event_type, 'origin': self.origin,
+                'timestamp': self.timestamp}
 
 
 class FinishReportingEvent(ReportingEvent):
 
-    def __init__(self, name, description, result=status.SUCCESS):
+    def __init__(self, name, description, result=status.SUCCESS,
+                 post_files=None):
         super(FinishReportingEvent, self).__init__(
             FINISH_EVENT_TYPE, name, description)
         self.result = result
+        if post_files is None:
+            post_files = []
+        self.post_files = post_files
         if result not in status:
             raise ValueError("Invalid result: %s" % result)
 
@@ -60,6 +73,8 @@ class FinishReportingEvent(ReportingEvent):
         """The event represented as json friendly."""
         data = super(FinishReportingEvent, self).as_dict()
         data['result'] = self.result
+        if self.post_files:
+            data['files'] = _collect_file_info(self.post_files)
         return data
 
 
@@ -78,12 +93,13 @@ def report_event(event):
 
 
 def report_finish_event(event_name, event_description,
-                        result=status.SUCCESS):
+                        result=status.SUCCESS, post_files=None):
     """Report a "finish" event.
 
     See :py:func:`.report_event` for parameter details.
     """
-    event = FinishReportingEvent(event_name, event_description, result)
+    event = FinishReportingEvent(event_name, event_description, result,
+                                 post_files=post_files)
     return report_event(event)
 
 
@@ -133,13 +149,17 @@ class ReportEventStack(object):
         value is FAIL.
     """
     def __init__(self, name, description, message=None, parent=None,
-                 reporting_enabled=None, result_on_exception=status.FAIL):
+                 reporting_enabled=None, result_on_exception=status.FAIL,
+                 post_files=None):
         self.parent = parent
         self.name = name
         self.description = description
         self.message = message
         self.result_on_exception = result_on_exception
         self.result = status.SUCCESS
+        if post_files is None:
+            post_files = []
+        self.post_files = post_files
 
         # use parents reporting value if not provided
         if reporting_enabled is None:
@@ -205,6 +225,22 @@ class ReportEventStack(object):
         if self.parent:
             self.parent.children[self.name] = (result, msg)
         if self.reporting_enabled:
-            report_finish_event(self.fullname, msg, result)
+            report_finish_event(self.fullname, msg, result,
+                                post_files=self.post_files)
+
+
+def _collect_file_info(files):
+    if not files:
+        return None
+    ret = []
+    for fname in files:
+        if not os.path.isfile(fname):
+            content = None
+        else:
+            with open(fname, "rb") as fp:
+                content = base64.b64encode(fp.read()).decode()
+        ret.append({'path': fname, 'content': content,
+                    'encoding': 'base64'})
+    return ret
 
-# vi: ts=4 expandtab
+# vi: ts=4 expandtab syntax=python
diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py
index bb67ef73..0a441adf 100644
--- a/tests/unittests/test_reporting.py
+++ b/tests/unittests/test_reporting.py
@@ -241,7 +241,8 @@ class TestReportingEventStack(TestCase):
         self.assertEqual(
             [mock.call('myname', 'mydesc')], report_start.call_args_list)
         self.assertEqual(
-            [mock.call('myname', 'mydesc', events.status.SUCCESS)],
+            [mock.call('myname', 'mydesc', events.status.SUCCESS,
+                       post_files=[])],
             report_finish.call_args_list)
 
     @mock.patch('cloudinit.reporting.events.report_finish_event')
@@ -256,7 +257,7 @@ class TestReportingEventStack(TestCase):
             pass
         self.assertEqual([mock.call(name, desc)], report_start.call_args_list)
         self.assertEqual(
-            [mock.call(name, desc, events.status.FAIL)],
+            [mock.call(name, desc, events.status.FAIL, post_files=[])],
             report_finish.call_args_list)
 
     @mock.patch('cloudinit.reporting.events.report_finish_event')
@@ -272,7 +273,7 @@ class TestReportingEventStack(TestCase):
             pass
         self.assertEqual([mock.call(name, desc)], report_start.call_args_list)
         self.assertEqual(
-            [mock.call(name, desc, events.status.WARN)],
+            [mock.call(name, desc, events.status.WARN, post_files=[])],
             report_finish.call_args_list)
 
     @mock.patch('cloudinit.reporting.events.report_start_event')
@@ -301,7 +302,7 @@ class TestReportingEventStack(TestCase):
                 child.result = events.status.WARN
 
         report_finish.assert_called_with(
-            "topname", "topdesc", events.status.WARN)
+            "topname", "topdesc", events.status.WARN, post_files=[])
 
     @mock.patch('cloudinit.reporting.events.report_finish_event')
     def test_message_used_in_finish(self, report_finish):
@@ -309,7 +310,8 @@ class TestReportingEventStack(TestCase):
                                      message="mymessage"):
             pass
         self.assertEqual(
-            [mock.call("myname", "mymessage", events.status.SUCCESS)],
+            [mock.call("myname", "mymessage", events.status.SUCCESS,
+                       post_files=[])],
             report_finish.call_args_list)
 
     @mock.patch('cloudinit.reporting.events.report_finish_event')
@@ -317,7 +319,8 @@ class TestReportingEventStack(TestCase):
         with events.ReportEventStack("myname", "mydesc") as c:
             c.message = "all good"
         self.assertEqual(
-            [mock.call("myname", "all good", events.status.SUCCESS)],
+            [mock.call("myname", "all good", events.status.SUCCESS,
+                       post_files=[])],
             report_finish.call_args_list)
 
     @mock.patch('cloudinit.reporting.events.report_start_event')
-- 
cgit v1.2.3


From 8bcccd07d1dbde74126e81967388d2e5a90fcfa7 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Mon, 31 Aug 2015 14:10:54 -0400
Subject: handlers: docstring fixups, and print actually do something

---
 cloudinit/reporting/handlers.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'cloudinit')

diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py
index 5ed3cb84..140a98c5 100644
--- a/cloudinit/reporting/handlers.py
+++ b/cloudinit/reporting/handlers.py
@@ -22,12 +22,11 @@ class ReportingHandler(object):
 
     @abc.abstractmethod
     def publish_event(self, event):
-        """Publish an event to the ``INFO`` log level."""
-        print(event.as_string())
+        """Publish an event."""
 
 
 class LogHandler(ReportingHandler):
-    """Publishes events to the cloud-init log at the ``INFO`` log level."""
+    """Publishes events to the cloud-init log at the ``DEBUG`` log level."""
 
     def __init__(self, level="DEBUG"):
         super(LogHandler, self).__init__()
@@ -43,15 +42,16 @@ class LogHandler(ReportingHandler):
         self.level = level
 
     def publish_event(self, event):
-        """Publish an event to the ``INFO`` log level."""
         logger = logging.getLogger(
             '.'.join(['cloudinit', 'reporting', event.event_type, event.name]))
         logger.log(self.level, event.as_string())
 
 
 class PrintHandler(ReportingHandler):
+    """Print the event as a string."""
+
     def publish_event(self, event):
-        """Publish an event to the ``INFO`` log level."""
+        print(event.as_string())
 
 
 class WebHookHandler(ReportingHandler):
-- 
cgit v1.2.3


From 3f2dddae6e8d5148bcf89c2b4e27975d1da77aea Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Mon, 31 Aug 2015 14:11:47 -0400
Subject: handlers: drop unused import

this import was left over from before we moved oauthlib into url_helper
---
 cloudinit/reporting/handlers.py | 1 -
 1 file changed, 1 deletion(-)

(limited to 'cloudinit')

diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py
index 140a98c5..ba480da0 100644
--- a/cloudinit/reporting/handlers.py
+++ b/cloudinit/reporting/handlers.py
@@ -1,7 +1,6 @@
 # vi: ts=4 expandtab
 
 import abc
-import oauthlib.oauth1 as oauth1
 import six
 
 from ..registry import DictRegistry
-- 
cgit v1.2.3