From e74d7752f1761c3a8d3c19877de4707d00c49d08 Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Mon, 21 Aug 2017 13:46:23 -0600
Subject: tools: Add tooling for basic cloud-init performance analysis.

This branch adds cloudinit-analyze into cloud-init proper. It adds an
"analyze" subcommand to the cloud-init command line utility for quick
performance assessment of cloud-init stages and events.

On a cloud-init configured instance, running "cloud-init analyze blame"
will now report which cloud-init events cost the most wall time. This
allows for quick assessment of the most costly stages of cloud-init.

This functionality is pulled from Ryan Harper's analyze work.

The cloudinit-analyze main script itself has been refactored a bit for
inclusion as a subcommand of cloud-init CLI. There will be a followup
branch at some point which will optionally instrument detailed strace
profiling, but that approach needs a bit more discussion first.

This branch also adds:
 * additional debugging topic to the sphinx-generated docs describing
   cloud-init analyze, dump and show as well as cloud-init single usage.
 * Updates the Makefile unittests target to include cloudinit directory
   because we now have unittests within that package.

LP: #1709761
---
 tests/unittests/test_cli.py | 87 +++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 84 insertions(+), 3 deletions(-)

(limited to 'tests/unittests/test_cli.py')

diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index 06f366b2..7780f164 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -31,9 +31,90 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
 
     def test_no_arguments_shows_error_message(self):
         exit_code = self._call_main()
-        self.assertIn('cloud-init: error: too few arguments',
-                      self.stderr.getvalue())
+        missing_subcommand_message = [
+            'too few arguments',  # python2.7 msg
+            'the following arguments are required: subcommand'  # python3 msg
+        ]
+        error = self.stderr.getvalue()
+        matches = ([msg in error for msg in missing_subcommand_message])
+        self.assertTrue(
+            any(matches), 'Did not find error message for missing subcommand')
         self.assertEqual(2, exit_code)
 
+    def test_all_subcommands_represented_in_help(self):
+        """All known subparsers are represented in the cloud-int help doc."""
+        self._call_main()
+        error = self.stderr.getvalue()
+        expected_subcommands = ['analyze', 'init', 'modules', 'single',
+                                'dhclient-hook', 'features']
+        for subcommand in expected_subcommands:
+            self.assertIn(subcommand, error)
 
-# vi: ts=4 expandtab
+    @mock.patch('cloudinit.cmd.main.status_wrapper')
+    def test_init_subcommand_parser(self, m_status_wrapper):
+        """The subcommand 'init' calls status_wrapper passing init."""
+        self._call_main(['cloud-init', 'init'])
+        (name, parseargs) = m_status_wrapper.call_args_list[0][0]
+        self.assertEqual('init', name)
+        self.assertEqual('init', parseargs.subcommand)
+        self.assertEqual('init', parseargs.action[0])
+        self.assertEqual('main_init', parseargs.action[1].__name__)
+
+    @mock.patch('cloudinit.cmd.main.status_wrapper')
+    def test_modules_subcommand_parser(self, m_status_wrapper):
+        """The subcommand 'modules' calls status_wrapper passing modules."""
+        self._call_main(['cloud-init', 'modules'])
+        (name, parseargs) = m_status_wrapper.call_args_list[0][0]
+        self.assertEqual('modules', name)
+        self.assertEqual('modules', parseargs.subcommand)
+        self.assertEqual('modules', parseargs.action[0])
+        self.assertEqual('main_modules', parseargs.action[1].__name__)
+
+    def test_analyze_subcommand_parser(self):
+        """The subcommand cloud-init analyze calls the correct subparser."""
+        self._call_main(['cloud-init', 'analyze'])
+        # These subcommands only valid for cloud-init analyze script
+        expected_subcommands = ['blame', 'show', 'dump']
+        error = self.stderr.getvalue()
+        for subcommand in expected_subcommands:
+            self.assertIn(subcommand, error)
+
+    @mock.patch('cloudinit.cmd.main.main_single')
+    def test_single_subcommand(self, m_main_single):
+        """The subcommand 'single' calls main_single with valid args."""
+        self._call_main(['cloud-init', 'single', '--name', 'cc_ntp'])
+        (name, parseargs) = m_main_single.call_args_list[0][0]
+        self.assertEqual('single', name)
+        self.assertEqual('single', parseargs.subcommand)
+        self.assertEqual('single', parseargs.action[0])
+        self.assertFalse(parseargs.debug)
+        self.assertFalse(parseargs.force)
+        self.assertIsNone(parseargs.frequency)
+        self.assertEqual('cc_ntp', parseargs.name)
+        self.assertFalse(parseargs.report)
+
+    @mock.patch('cloudinit.cmd.main.dhclient_hook')
+    def test_dhclient_hook_subcommand(self, m_dhclient_hook):
+        """The subcommand 'dhclient-hook' calls dhclient_hook with args."""
+        self._call_main(['cloud-init', 'dhclient-hook', 'net_action', 'eth0'])
+        (name, parseargs) = m_dhclient_hook.call_args_list[0][0]
+        self.assertEqual('dhclient_hook', name)
+        self.assertEqual('dhclient-hook', parseargs.subcommand)
+        self.assertEqual('dhclient_hook', parseargs.action[0])
+        self.assertFalse(parseargs.debug)
+        self.assertFalse(parseargs.force)
+        self.assertEqual('net_action', parseargs.net_action)
+        self.assertEqual('eth0', parseargs.net_interface)
+
+    @mock.patch('cloudinit.cmd.main.main_features')
+    def test_features_hook_subcommand(self, m_features):
+        """The subcommand 'features' calls main_features with args."""
+        self._call_main(['cloud-init', 'features'])
+        (name, parseargs) = m_features.call_args_list[0][0]
+        self.assertEqual('features', name)
+        self.assertEqual('features', parseargs.subcommand)
+        self.assertEqual('features', parseargs.action[0])
+        self.assertFalse(parseargs.debug)
+        self.assertFalse(parseargs.force)
+
+# : ts=4 expandtab
-- 
cgit v1.2.3


From cc9762a2d737ead386ffb9f067adc5e543224560 Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Tue, 22 Aug 2017 20:06:20 -0600
Subject: schema cli: Add schema subcommand to cloud-init cli and cc_runcmd
 schema

This branch does a few things:
  - Add 'schema' subcommand to cloud-init CLI for validating
    cloud-config files against strict module jsonschema definitions
  - Add --annotate parameter to 'cloud-init schema' to annotate
    existing cloud-config file content with validation errors
  - Add jsonschema definition to cc_runcmd
  - Add unit test coverage for cc_runcmd
  - Update CLI capabilities documentation

This branch only imports development (and analyze) subparsers when the
specific subcommand is provided on the CLI to avoid adding costly unused
file imports during cloud-init system boot.

The schema command allows a person to quickly validate a cloud-config text
file against cloud-init's known module schemas to avoid costly roundtrips
deploying instances in their cloud of choice. As of this branch, only
cc_ntp and cc_runcmd cloud-config modules define schemas. Schema
validation will ignore all undefined config keys until all modules define
a strict schema.

To perform validation of runcmd and ntp sections of a cloud-config file:
$ cat > cloud.cfg <<EOF
runcmd: bogus
EOF
$ python -m cloudinit.cmd.main schema --config-file cloud.cfg

$ python -m cloudinit.cmd.main schema --config-file cloud.cfg \
  --annotate

Once jsonschema is defined for all ~55 cc modules, we will move this
schema subcommand up as a proper subcommand of the cloud-init CLI.
---
 cloudinit/cmd/devel/__init__.py                    |   0
 cloudinit/cmd/devel/parser.py                      |  26 +++
 cloudinit/cmd/main.py                              |  21 ++-
 cloudinit/config/cc_runcmd.py                      |  82 ++++++---
 cloudinit/config/schema.py                         | 199 +++++++++++++++++----
 doc/rtd/topics/capabilities.rst                    |  18 +-
 tests/unittests/test_cli.py                        |  21 ++-
 .../unittests/test_handler/test_handler_runcmd.py  | 108 +++++++++++
 tests/unittests/test_handler/test_schema.py        | 157 ++++++++++++++--
 9 files changed, 541 insertions(+), 91 deletions(-)
 create mode 100644 cloudinit/cmd/devel/__init__.py
 create mode 100644 cloudinit/cmd/devel/parser.py
 create mode 100644 tests/unittests/test_handler/test_handler_runcmd.py

(limited to 'tests/unittests/test_cli.py')

diff --git a/cloudinit/cmd/devel/__init__.py b/cloudinit/cmd/devel/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/cloudinit/cmd/devel/parser.py b/cloudinit/cmd/devel/parser.py
new file mode 100644
index 00000000..acacc4ed
--- /dev/null
+++ b/cloudinit/cmd/devel/parser.py
@@ -0,0 +1,26 @@
+# Copyright (C) 2017 Canonical Ltd.
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Define 'devel' subcommand argument parsers to include in cloud-init cmd."""
+
+import argparse
+from cloudinit.config.schema import (
+    get_parser as schema_parser, handle_schema_args)
+
+
+def get_parser(parser=None):
+    if not parser:
+        parser = argparse.ArgumentParser(
+            prog='cloudinit-devel',
+            description='Run development cloud-init tools')
+    subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand')
+    subparsers.required = True
+
+    parser_schema = subparsers.add_parser(
+        'schema', help='Validate cloud-config files or document schema')
+    # Construct schema subcommand parser
+    schema_parser(parser_schema)
+    parser_schema.set_defaults(action=('schema', handle_schema_args))
+
+    return parser
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 9c0ac864..5b467979 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -705,7 +705,6 @@ def main(sysv_args=None):
     subparsers.required = True
 
     # Each action and its sub-options (if any)
-
     parser_init = subparsers.add_parser('init',
                                         help=('initializes cloud-init and'
                                               ' performs initial modules'))
@@ -762,12 +761,20 @@ def main(sysv_args=None):
 
     parser_analyze = subparsers.add_parser(
         'analyze', help='Devel tool: Analyze cloud-init logs and data')
-    if sysv_args and sysv_args[0] == 'analyze':
-        # Only load this parser if analyze is specified to avoid file load cost
-        # FIXME put this under 'devel' subcommand (coming in next branch)
-        from cloudinit.analyze.__main__ import get_parser as analyze_parser
-        # Construct analyze subcommand parser
-        analyze_parser(parser_analyze)
+
+    parser_devel = subparsers.add_parser(
+        'devel', help='Run development tools')
+
+    if sysv_args:
+        # Only load subparsers if subcommand is specified to avoid load cost
+        if sysv_args[0] == 'analyze':
+            from cloudinit.analyze.__main__ import get_parser as analyze_parser
+            # Construct analyze subcommand parser
+            analyze_parser(parser_analyze)
+        if sysv_args[0] == 'devel':
+            from cloudinit.cmd.devel.parser import get_parser as devel_parser
+            # Construct devel subcommand parser
+            devel_parser(parser_devel)
 
     args = parser.parse_args(args=sysv_args)
 
diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py
index dfa8cb3d..7c3ccd41 100644
--- a/cloudinit/config/cc_runcmd.py
+++ b/cloudinit/config/cc_runcmd.py
@@ -6,41 +6,66 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""
-Runcmd
-------
-**Summary:** run commands
+"""Runcmd: run arbitrary commands at rc.local with output to the console"""
 
-Run arbitrary commands at a rc.local like level with output to the console.
-Each item can be either a list or a string. If the item is a list, it will be
-properly executed as if passed to ``execve()`` (with the first arg as the
-command). If the item is a string, it will be written to a file and interpreted
-using ``sh``.
-
-.. note::
-    all commands must be proper yaml, so you have to quote any characters yaml
-    would eat (':' can be problematic)
-
-**Internal name:** ``cc_runcmd``
-
-**Module frequency:** per instance
+from cloudinit.config.schema import validate_cloudconfig_schema
+from cloudinit.settings import PER_INSTANCE
+from cloudinit import util
 
-**Supported distros:** all
+import os
+from textwrap import dedent
 
-**Config keys**::
 
-    runcmd:
-        - [ ls, -l, / ]
-        - [ sh, -xc, "echo $(date) ': hello world!'" ]
-        - [ sh, -c, echo "=========hello world'=========" ]
-        - ls -l /root
-        - [ wget, "http://example.org", -O, /tmp/index.html ]
-"""
+# The schema definition for each cloud-config module is a strict contract for
+# describing supported configuration parameters for each cloud-config section.
+# It allows cloud-config to validate and alert users to invalid or ignored
+# configuration options before actually attempting to deploy with said
+# configuration.
 
+distros = ['all']
 
-import os
+schema = {
+    'id': 'cc_runcmd',
+    'name': 'Runcmd',
+    'title': 'Run arbitrary commands',
+    'description': dedent("""\
+        Run arbitrary commands at a rc.local like level with output to the
+        console. Each item can be either a list or a string. If the item is a
+        list, it will be properly executed as if passed to ``execve()`` (with
+        the first arg as the command). If the item is a string, it will be
+        written to a file and interpreted
+        using ``sh``.
 
-from cloudinit import util
+        .. note::
+        all commands must be proper yaml, so you have to quote any characters
+        yaml would eat (':' can be problematic)"""),
+    'distros': distros,
+    'examples': [dedent("""\
+        runcmd:
+            - [ ls, -l, / ]
+            - [ sh, -xc, "echo $(date) ': hello world!'" ]
+            - [ sh, -c, echo "=========hello world'=========" ]
+            - ls -l /root
+            - [ wget, "http://example.org", -O, /tmp/index.html ]
+    """)],
+    'frequency': PER_INSTANCE,
+    'type': 'object',
+    'properties': {
+        'runcmd': {
+            'type': 'array',
+            'items': {
+                'oneOf': [
+                    {'type': 'array', 'items': {'type': 'string'}},
+                    {'type': 'string'}]
+            },
+            'additionalItems': False,  # Reject items of non-string non-list
+            'additionalProperties': False,
+            'minItems': 1,
+            'required': [],
+            'uniqueItems': True
+        }
+    }
+}
 
 
 def handle(name, cfg, cloud, log, _args):
@@ -49,6 +74,7 @@ def handle(name, cfg, cloud, log, _args):
                    " no 'runcmd' key in configuration"), name)
         return
 
+    validate_cloudconfig_schema(cfg, schema)
     out_fn = os.path.join(cloud.get_ipath('scripts'), "runcmd")
     cmd = cfg["runcmd"]
     try:
diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py
index 6400f005..73dd5c2e 100644
--- a/cloudinit/config/schema.py
+++ b/cloudinit/config/schema.py
@@ -3,11 +3,14 @@
 
 from __future__ import print_function
 
-from cloudinit.util import read_file_or_url
+from cloudinit import importer
+from cloudinit.util import find_modules, read_file_or_url
 
 import argparse
+from collections import defaultdict
 import logging
 import os
+import re
 import sys
 import yaml
 
@@ -15,7 +18,7 @@ SCHEMA_UNDEFINED = b'UNDEFINED'
 CLOUD_CONFIG_HEADER = b'#cloud-config'
 SCHEMA_DOC_TMPL = """
 {name}
----
+{title_underbar}
 **Summary:** {title}
 
 {description}
@@ -83,11 +86,49 @@ def validate_cloudconfig_schema(config, schema, strict=False):
             logging.warning('Invalid config:\n%s', '\n'.join(messages))
 
 
-def validate_cloudconfig_file(config_path, schema):
+def annotated_cloudconfig_file(cloudconfig, original_content, schema_errors):
+    """Return contents of the cloud-config file annotated with schema errors.
+
+    @param cloudconfig: YAML-loaded object from the original_content.
+    @param original_content: The contents of a cloud-config file
+    @param schema_errors: List of tuples from a JSONSchemaValidationError. The
+        tuples consist of (schemapath, error_message).
+    """
+    if not schema_errors:
+        return original_content
+    schemapaths = _schemapath_for_cloudconfig(cloudconfig, original_content)
+    errors_by_line = defaultdict(list)
+    error_count = 1
+    error_footer = []
+    annotated_content = []
+    for path, msg in schema_errors:
+        errors_by_line[schemapaths[path]].append(msg)
+        error_footer.append('# E{0}: {1}'.format(error_count, msg))
+        error_count += 1
+    lines = original_content.decode().split('\n')
+    error_count = 1
+    for line_number, line in enumerate(lines):
+        errors = errors_by_line[line_number + 1]
+        if errors:
+            error_label = ','.join(
+                ['E{0}'.format(count + error_count)
+                 for count in range(0, len(errors))])
+            error_count += len(errors)
+            annotated_content.append(line + '\t\t# ' + error_label)
+        else:
+            annotated_content.append(line)
+    annotated_content.append(
+        '# Errors: -------------\n{0}\n\n'.format('\n'.join(error_footer)))
+    return '\n'.join(annotated_content)
+
+
+def validate_cloudconfig_file(config_path, schema, annotate=False):
     """Validate cloudconfig file adheres to a specific jsonschema.
 
     @param config_path: Path to the yaml cloud-config file to parse.
     @param schema: Dict describing a valid jsonschema to validate against.
+    @param annotate: Boolean set True to print original config file with error
+        annotations on the offending lines.
 
     @raises SchemaValidationError containing any of schema_errors encountered.
     @raises RuntimeError when config_path does not exist.
@@ -108,8 +149,64 @@ def validate_cloudconfig_file(config_path, schema):
             ('format', 'File {0} is not valid yaml. {1}'.format(
                 config_path, str(e))),)
         raise SchemaValidationError(errors)
-    validate_cloudconfig_schema(
-        cloudconfig, schema, strict=True)
+
+    try:
+        validate_cloudconfig_schema(
+            cloudconfig, schema, strict=True)
+    except SchemaValidationError as e:
+        if annotate:
+            print(annotated_cloudconfig_file(
+                cloudconfig, content, e.schema_errors))
+        raise
+
+
+def _schemapath_for_cloudconfig(config, original_content):
+    """Return a dictionary mapping schemapath to original_content line number.
+
+    @param config: The yaml.loaded config dictionary of a cloud-config file.
+    @param original_content: The simple file content of the cloud-config file
+    """
+    # FIXME Doesn't handle multi-line lists or multi-line strings
+    content_lines = original_content.decode().split('\n')
+    schema_line_numbers = {}
+    list_index = 0
+    RE_YAML_INDENT = r'^(\s*)'
+    scopes = []
+    for line_number, line in enumerate(content_lines):
+        indent_depth = len(re.match(RE_YAML_INDENT, line).groups()[0])
+        line = line.strip()
+        if not line or line.startswith('#'):
+            continue
+        if scopes:
+            previous_depth, path_prefix = scopes[-1]
+        else:
+            previous_depth = -1
+            path_prefix = ''
+        if line.startswith('- '):
+            key = str(list_index)
+            value = line[1:]
+            list_index += 1
+        else:
+            list_index = 0
+            key, value = line.split(':', 1)
+        while indent_depth <= previous_depth:
+            if scopes:
+                previous_depth, path_prefix = scopes.pop()
+            else:
+                previous_depth = -1
+                path_prefix = ''
+        if path_prefix:
+            key = path_prefix + '.' + key
+        scopes.append((indent_depth, key))
+        if value:
+            value = value.strip()
+            if value.startswith('['):
+                scopes.append((indent_depth + 2, key + '.0'))
+                for inner_list_index in range(0, len(yaml.safe_load(value))):
+                    list_key = key + '.' + str(inner_list_index)
+                    schema_line_numbers[list_key] = line_number + 1
+        schema_line_numbers[key] = line_number + 1
+    return schema_line_numbers
 
 
 def _get_property_type(property_dict):
@@ -117,9 +214,15 @@ def _get_property_type(property_dict):
     property_type = property_dict.get('type', SCHEMA_UNDEFINED)
     if isinstance(property_type, list):
         property_type = '/'.join(property_type)
-    item_type = property_dict.get('items', {}).get('type')
-    if item_type:
-        property_type = '{0} of {1}'.format(property_type, item_type)
+    items = property_dict.get('items', {})
+    sub_property_type = items.get('type', '')
+    # Collect each item type
+    for sub_item in items.get('oneOf', {}):
+        if sub_property_type:
+            sub_property_type += '/'
+        sub_property_type += '(' + _get_property_type(sub_item) + ')'
+    if sub_property_type:
+        return '{0} of {1}'.format(property_type, sub_property_type)
     return property_type
 
 
@@ -148,9 +251,12 @@ def _get_schema_examples(schema, prefix=''):
         return ''
     rst_content = '\n**Examples**::\n\n'
     for example in examples:
-        example_yaml = yaml.dump(example, default_flow_style=False)
+        if isinstance(example, str):
+            example_content = example
+        else:
+            example_content = yaml.dump(example, default_flow_style=False)
         # Python2.6 is missing textwrapper.indent
-        lines = example_yaml.split('\n')
+        lines = example_content.split('\n')
         indented_lines = ['    {0}'.format(line) for line in lines]
         rst_content += '\n'.join(indented_lines)
     return rst_content
@@ -165,58 +271,83 @@ def get_schema_doc(schema):
     schema['property_doc'] = _get_property_doc(schema)
     schema['examples'] = _get_schema_examples(schema)
     schema['distros'] = ', '.join(schema['distros'])
+    # Need an underbar of the same length as the name
+    schema['title_underbar'] = re.sub(r'.', '-', schema['name'])
     return SCHEMA_DOC_TMPL.format(**schema)
 
 
-def get_schema(section_key=None):
-    """Return a dict of jsonschema defined in any cc_* module.
+FULL_SCHEMA = None
 
-    @param: section_key: Optionally limit schema to a specific top-level key.
-    """
-    # TODO use util.find_modules in subsequent branch
-    from cloudinit.config.cc_ntp import schema
-    return schema
+
+def get_schema():
+    """Return jsonschema coalesced from all cc_* cloud-config module."""
+    global FULL_SCHEMA
+    if FULL_SCHEMA:
+        return FULL_SCHEMA
+    full_schema = {
+        '$schema': 'http://json-schema.org/draft-04/schema#',
+        'id': 'cloud-config-schema', 'allOf': []}
+
+    configs_dir = os.path.dirname(os.path.abspath(__file__))
+    potential_handlers = find_modules(configs_dir)
+    for (fname, mod_name) in potential_handlers.items():
+        mod_locs, looked_locs = importer.find_module(
+            mod_name, ['cloudinit.config'], ['schema'])
+        if mod_locs:
+            mod = importer.import_module(mod_locs[0])
+            full_schema['allOf'].append(mod.schema)
+    FULL_SCHEMA = full_schema
+    return full_schema
 
 
 def error(message):
     print(message, file=sys.stderr)
-    return 1
+    sys.exit(1)
 
 
-def get_parser():
+def get_parser(parser=None):
     """Return a parser for supported cmdline arguments."""
-    parser = argparse.ArgumentParser()
+    if not parser:
+        parser = argparse.ArgumentParser(
+            prog='cloudconfig-schema',
+            description='Validate cloud-config files or document schema')
     parser.add_argument('-c', '--config-file',
                         help='Path of the cloud-config yaml file to validate')
     parser.add_argument('-d', '--doc', action="store_true", default=False,
                         help='Print schema documentation')
-    parser.add_argument('-k', '--key',
-                        help='Limit validation or docs to a section key')
+    parser.add_argument('--annotate', action="store_true", default=False,
+                        help='Annotate existing cloud-config file with errors')
     return parser
 
 
-def main():
-    """Tool to validate schema of a cloud-config file or print schema docs."""
-    parser = get_parser()
-    args = parser.parse_args()
+def handle_schema_args(name, args):
+    """Handle provided schema args and perform the appropriate actions."""
     exclusive_args = [args.config_file, args.doc]
     if not any(exclusive_args) or all(exclusive_args):
-        return error('Expected either --config-file argument or --doc')
-
-    schema = get_schema()
+        error('Expected either --config-file argument or --doc')
+    full_schema = get_schema()
     if args.config_file:
         try:
-            validate_cloudconfig_file(args.config_file, schema)
+            validate_cloudconfig_file(
+                args.config_file, full_schema, args.annotate)
         except (SchemaValidationError, RuntimeError) as e:
-            return error(str(e))
-        print("Valid cloud-config file {0}".format(args.config_file))
+            if not args.annotate:
+                error(str(e))
+        else:
+            print("Valid cloud-config file {0}".format(args.config_file))
     if args.doc:
-        print(get_schema_doc(schema))
+        for subschema in full_schema['allOf']:
+            print(get_schema_doc(subschema))
+
+
+def main():
+    """Tool to validate schema of a cloud-config file or print schema docs."""
+    parser = get_parser()
+    handle_schema_args('cloudconfig-schema', parser.parse_args())
     return 0
 
 
 if __name__ == '__main__':
     sys.exit(main())
 
-
 # vi: ts=4 expandtab
diff --git a/doc/rtd/topics/capabilities.rst b/doc/rtd/topics/capabilities.rst
index b8034b07..31eaba53 100644
--- a/doc/rtd/topics/capabilities.rst
+++ b/doc/rtd/topics/capabilities.rst
@@ -51,15 +51,6 @@ described in this document.
   usage: cloud-init [-h] [--version] [--file FILES] [--debug] [--force]
                     {init,modules,query,single,dhclient-hook,features} ...
 
-  positional arguments:
-    {init,modules,query,single,dhclient-hook,features}
-      init                initializes cloud-init and performs initial modules
-      modules             activates modules using a given configuration key
-      query               query information stored in cloud-init
-      single              run a single module
-      dhclient-hook       run the dhclient hookto record network info
-      features            list defined features
-
   optional arguments:
     -h, --help            show this help message and exit
     --version, -v         show program's version number and exit
@@ -69,6 +60,15 @@ described in this document.
     --force               force running even if no datasource is found (use at
                           your own risk)
 
+  Subcommands:
+    {init,modules,single,dhclient-hook,features,analyze,devel}
+      init                initializes cloud-init and performs initial modules
+      modules             activates modules using a given configuration key
+      single              run a single module
+      dhclient-hook       run the dhclient hookto record network info
+      features            list defined features
+      analyze             Devel tool: Analyze cloud-init logs and data
+      devel               Run development tools
 
   % cloud-init features
   NETWORK_CONFIG_V1
diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index 7780f164..24498802 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -46,7 +46,7 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
         self._call_main()
         error = self.stderr.getvalue()
         expected_subcommands = ['analyze', 'init', 'modules', 'single',
-                                'dhclient-hook', 'features']
+                                'dhclient-hook', 'features', 'devel']
         for subcommand in expected_subcommands:
             self.assertIn(subcommand, error)
 
@@ -79,6 +79,25 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
         for subcommand in expected_subcommands:
             self.assertIn(subcommand, error)
 
+    def test_devel_subcommand_parser(self):
+        """The subcommand cloud-init devel calls the correct subparser."""
+        self._call_main(['cloud-init', 'devel'])
+        # These subcommands only valid for cloud-init schema script
+        expected_subcommands = ['schema']
+        error = self.stderr.getvalue()
+        for subcommand in expected_subcommands:
+            self.assertIn(subcommand, error)
+
+    @mock.patch('cloudinit.config.schema.handle_schema_args')
+    def test_wb_devel_schema_subcommand_parser(self, m_schema):
+        """The subcommand cloud-init schema calls the correct subparser."""
+        exit_code = self._call_main(['cloud-init', 'devel', 'schema'])
+        self.assertEqual(1, exit_code)
+        # Known whitebox output from schema subcommand
+        self.assertEqual(
+            'Expected either --config-file argument or --doc\n',
+            self.stderr.getvalue())
+
     @mock.patch('cloudinit.cmd.main.main_single')
     def test_single_subcommand(self, m_main_single):
         """The subcommand 'single' calls main_single with valid args."""
diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py
new file mode 100644
index 00000000..7880ee72
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_runcmd.py
@@ -0,0 +1,108 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit.config import cc_runcmd
+from cloudinit.sources import DataSourceNone
+from cloudinit import (distros, helpers, cloud, util)
+from ..helpers import FilesystemMockingTestCase, skipIf
+
+import logging
+import os
+import stat
+
+try:
+    import jsonschema
+    assert jsonschema  # avoid pyflakes error F401: import unused
+    _missing_jsonschema_dep = False
+except ImportError:
+    _missing_jsonschema_dep = True
+
+LOG = logging.getLogger(__name__)
+
+
+class TestRuncmd(FilesystemMockingTestCase):
+
+    with_logs = True
+
+    def setUp(self):
+        super(TestRuncmd, self).setUp()
+        self.subp = util.subp
+        self.new_root = self.tmp_dir()
+
+    def _get_cloud(self, distro):
+        self.patchUtils(self.new_root)
+        paths = helpers.Paths({'scripts': self.new_root})
+        cls = distros.fetch(distro)
+        mydist = cls(distro, {}, paths)
+        myds = DataSourceNone.DataSourceNone({}, mydist, paths)
+        paths.datasource = myds
+        return cloud.Cloud(myds, paths, {}, mydist, None)
+
+    def test_handler_skip_if_no_runcmd(self):
+        """When the provided config doesn't contain runcmd, skip it."""
+        cfg = {}
+        mycloud = self._get_cloud('ubuntu')
+        cc_runcmd.handle('notimportant', cfg, mycloud, LOG, None)
+        self.assertIn(
+            "Skipping module named notimportant, no 'runcmd' key",
+            self.logs.getvalue())
+
+    def test_handler_invalid_command_set(self):
+        """Commands which can't be converted to shell will raise errors."""
+        invalid_config = {'runcmd': 1}
+        cc = self._get_cloud('ubuntu')
+        cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, [])
+        self.assertIn(
+            'Failed to shellify 1 into file'
+            ' /var/lib/cloud/instances/iid-datasource-none/scripts/runcmd',
+            self.logs.getvalue())
+
+    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    def test_handler_schema_validation_warns_non_array_type(self):
+        """Schema validation warns of non-array type for runcmd key.
+
+        Schema validation is not strict, so runcmd attempts to shellify the
+        invalid content.
+        """
+        invalid_config = {'runcmd': 1}
+        cc = self._get_cloud('ubuntu')
+        cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, [])
+        self.assertIn(
+            'Invalid config:\nruncmd: 1 is not of type \'array\'',
+            self.logs.getvalue())
+        self.assertIn('Failed to shellify', self.logs.getvalue())
+
+    @skipIf(_missing_jsonschema_dep, 'No python-jsonschema dependency')
+    def test_handler_schema_validation_warns_non_array_item_type(self):
+        """Schema validation warns of non-array or string runcmd items.
+
+        Schema validation is not strict, so runcmd attempts to shellify the
+        invalid content.
+        """
+        invalid_config = {
+            'runcmd': ['ls /', 20, ['wget', 'http://stuff/blah'], {'a': 'n'}]}
+        cc = self._get_cloud('ubuntu')
+        cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, [])
+        expected_warnings = [
+            'runcmd.1: 20 is not valid under any of the given schemas',
+            'runcmd.3: {\'a\': \'n\'} is not valid under any of the given'
+            ' schema'
+        ]
+        logs = self.logs.getvalue()
+        for warning in expected_warnings:
+            self.assertIn(warning, logs)
+        self.assertIn('Failed to shellify', logs)
+
+    def test_handler_write_valid_runcmd_schema_to_file(self):
+        """Valid runcmd schema is written to a runcmd shell script."""
+        valid_config = {'runcmd': [['ls', '/']]}
+        cc = self._get_cloud('ubuntu')
+        cc_runcmd.handle('cc_runcmd', valid_config, cc, LOG, [])
+        runcmd_file = os.path.join(
+            self.new_root,
+            'var/lib/cloud/instances/iid-datasource-none/scripts/runcmd')
+        self.assertEqual("#!/bin/sh\n'ls' '/'\n", util.load_file(runcmd_file))
+        file_stat = os.stat(runcmd_file)
+        self.assertEqual(0o700, stat.S_IMODE(file_stat.st_mode))
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index eda4802a..640f11d4 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -1,9 +1,9 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit.config.schema import (
-    CLOUD_CONFIG_HEADER, SchemaValidationError, get_schema_doc,
-    validate_cloudconfig_file, validate_cloudconfig_schema,
-    main)
+    CLOUD_CONFIG_HEADER, SchemaValidationError, annotated_cloudconfig_file,
+    get_schema_doc, get_schema, validate_cloudconfig_file,
+    validate_cloudconfig_schema, main)
 from cloudinit.util import write_file
 
 from ..helpers import CiTestCase, mock, skipIf
@@ -11,6 +11,7 @@ from ..helpers import CiTestCase, mock, skipIf
 from copy import copy
 from six import StringIO
 from textwrap import dedent
+from yaml import safe_load
 
 try:
     import jsonschema
@@ -20,6 +21,29 @@ except ImportError:
     _missing_jsonschema_dep = True
 
 
+class GetSchemaTest(CiTestCase):
+
+    def test_get_schema_coalesces_known_schema(self):
+        """Every cloudconfig module with schema is listed in allOf keyword."""
+        schema = get_schema()
+        self.assertItemsEqual(
+            ['cc_ntp', 'cc_runcmd'],
+            [subschema['id'] for subschema in schema['allOf']])
+        self.assertEqual('cloud-config-schema', schema['id'])
+        self.assertEqual(
+            'http://json-schema.org/draft-04/schema#',
+            schema['$schema'])
+        # FULL_SCHEMA is updated by the get_schema call
+        from cloudinit.config.schema import FULL_SCHEMA
+        self.assertItemsEqual(['id', '$schema', 'allOf'], FULL_SCHEMA.keys())
+
+    def test_get_schema_returns_global_when_set(self):
+        """When FULL_SCHEMA global is already set, get_schema returns it."""
+        m_schema_path = 'cloudinit.config.schema.FULL_SCHEMA'
+        with mock.patch(m_schema_path, {'here': 'iam'}):
+            self.assertEqual({'here': 'iam'}, get_schema())
+
+
 class SchemaValidationErrorTest(CiTestCase):
     """Test validate_cloudconfig_schema"""
 
@@ -151,11 +175,11 @@ class GetSchemaDocTest(CiTestCase):
         full_schema.update(
             {'properties': {
                 'prop1': {'type': 'array', 'description': 'prop-description',
-                          'items': {'type': 'int'}}}})
+                          'items': {'type': 'integer'}}}})
         self.assertEqual(
             dedent("""
                 name
-                ---
+                ----
                 **Summary:** title
 
                 description
@@ -167,27 +191,71 @@ class GetSchemaDocTest(CiTestCase):
                 **Supported distros:** debian, rhel
 
                 **Config schema**:
-                    **prop1:** (array of int) prop-description\n\n"""),
+                    **prop1:** (array of integer) prop-description\n\n"""),
+            get_schema_doc(full_schema))
+
+    def test_get_schema_doc_handles_multiple_types(self):
+        """get_schema_doc delimits multiple property types with a '/'."""
+        full_schema = copy(self.required_schema)
+        full_schema.update(
+            {'properties': {
+                'prop1': {'type': ['string', 'integer'],
+                          'description': 'prop-description'}}})
+        self.assertIn(
+            '**prop1:** (string/integer) prop-description',
+            get_schema_doc(full_schema))
+
+    def test_get_schema_doc_handles_nested_oneof_property_types(self):
+        """get_schema_doc describes array items oneOf declarations in type."""
+        full_schema = copy(self.required_schema)
+        full_schema.update(
+            {'properties': {
+                'prop1': {'type': 'array',
+                          'items': {
+                              'oneOf': [{'type': 'string'},
+                                        {'type': 'integer'}]},
+                          'description': 'prop-description'}}})
+        self.assertIn(
+            '**prop1:** (array of (string)/(integer)) prop-description',
             get_schema_doc(full_schema))
 
     def test_get_schema_doc_returns_restructured_text_with_examples(self):
         """get_schema_doc returns indented examples when present in schema."""
         full_schema = copy(self.required_schema)
         full_schema.update(
-            {'examples': {'ex1': [1, 2, 3]},
+            {'examples': [{'ex1': [1, 2, 3]}],
              'properties': {
                 'prop1': {'type': 'array', 'description': 'prop-description',
-                          'items': {'type': 'int'}}}})
+                          'items': {'type': 'integer'}}}})
         self.assertIn(
             dedent("""
                 **Config schema**:
-                    **prop1:** (array of int) prop-description
+                    **prop1:** (array of integer) prop-description
 
                 **Examples**::
 
                     ex1"""),
             get_schema_doc(full_schema))
 
+    def test_get_schema_doc_handles_unstructured_examples(self):
+        """get_schema_doc properly indented examples which as just strings."""
+        full_schema = copy(self.required_schema)
+        full_schema.update(
+            {'examples': ['My example:\n    [don\'t, expand, "this"]'],
+             'properties': {
+                'prop1': {'type': 'array', 'description': 'prop-description',
+                          'items': {'type': 'integer'}}}})
+        self.assertIn(
+            dedent("""
+                **Config schema**:
+                    **prop1:** (array of integer) prop-description
+
+                **Examples**::
+
+                    My example:
+                        [don't, expand, "this"]"""),
+            get_schema_doc(full_schema))
+
     def test_get_schema_doc_raises_key_errors(self):
         """get_schema_doc raises KeyErrors on missing keys."""
         for key in self.required_schema:
@@ -198,13 +266,78 @@ class GetSchemaDocTest(CiTestCase):
             self.assertIn(key, str(context_mgr.exception))
 
 
+class AnnotatedCloudconfigFileTest(CiTestCase):
+    maxDiff = None
+
+    def test_annotated_cloudconfig_file_no_schema_errors(self):
+        """With no schema_errors, print the original content."""
+        content = b'ntp:\n  pools: [ntp1.pools.com]\n'
+        self.assertEqual(
+            content,
+            annotated_cloudconfig_file({}, content, schema_errors=[]))
+
+    def test_annotated_cloudconfig_file_schema_annotates_and_adds_footer(self):
+        """With schema_errors, error lines are annotated and a footer added."""
+        content = dedent("""\
+            #cloud-config
+            # comment
+            ntp:
+              pools: [-99, 75]
+            """).encode()
+        expected = dedent("""\
+            #cloud-config
+            # comment
+            ntp:		# E1
+              pools: [-99, 75]		# E2,E3
+
+            # Errors: -------------
+            # E1: Some type error
+            # E2: -99 is not a string
+            # E3: 75 is not a string
+
+            """)
+        parsed_config = safe_load(content[13:])
+        schema_errors = [
+            ('ntp', 'Some type error'), ('ntp.pools.0', '-99 is not a string'),
+            ('ntp.pools.1', '75 is not a string')]
+        self.assertEqual(
+            expected,
+            annotated_cloudconfig_file(parsed_config, content, schema_errors))
+
+    def test_annotated_cloudconfig_file_annotates_separate_line_items(self):
+        """Errors are annotated for lists with items on separate lines."""
+        content = dedent("""\
+            #cloud-config
+            # comment
+            ntp:
+              pools:
+                - -99
+                - 75
+            """).encode()
+        expected = dedent("""\
+            ntp:
+              pools:
+                - -99		# E1
+                - 75		# E2
+            """)
+        parsed_config = safe_load(content[13:])
+        schema_errors = [
+            ('ntp.pools.0', '-99 is not a string'),
+            ('ntp.pools.1', '75 is not a string')]
+        self.assertIn(
+            expected,
+            annotated_cloudconfig_file(parsed_config, content, schema_errors))
+
+
 class MainTest(CiTestCase):
 
     def test_main_missing_args(self):
         """Main exits non-zero and reports an error on missing parameters."""
         with mock.patch('sys.argv', ['mycmd']):
             with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
-                self.assertEqual(1, main(), 'Expected non-zero exit code')
+                with self.assertRaises(SystemExit) as context_manager:
+                    main()
+        self.assertEqual('1', str(context_manager.exception))
         self.assertEqual(
             'Expected either --config-file argument or --doc\n',
             m_stderr.getvalue())
@@ -216,13 +349,13 @@ class MainTest(CiTestCase):
             with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
                 self.assertEqual(0, main(), 'Expected 0 exit code')
         self.assertIn('\nNTP\n---\n', m_stdout.getvalue())
+        self.assertIn('\nRuncmd\n------\n', m_stdout.getvalue())
 
     def test_main_validates_config_file(self):
         """When --config-file parameter is provided, main validates schema."""
         myyaml = self.tmp_path('my.yaml')
         myargs = ['mycmd', '--config-file', myyaml]
-        with open(myyaml, 'wb') as stream:
-            stream.write(b'#cloud-config\nntp:')   # shortest ntp schema
+        write_file(myyaml, b'#cloud-config\nntp:')  # shortest ntp schema
         with mock.patch('sys.argv', myargs):
             with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
                 self.assertEqual(0, main(), 'Expected 0 exit code')
-- 
cgit v1.2.3


From 89579a68d9f51e51b24f96b933da656afd83edfb Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Wed, 23 Aug 2017 18:54:01 -0600
Subject: cli: Fix command line parsing of coniditionally loaded subcommands.

In an effort to save file load cost during system boot, certain
subcommands, analyze and devel, do not get loaded unless the subcommand is
specified on the commandline. Because setup.py entrypoint for cloud-init
script doesn't specify sysv_args parameter when calling the CLI's main()
we need main to read sys.argv into sysv_args so our subparser loading
continues to work.

LP: #1712676
---
 cloudinit/cmd/main.py       |  9 ++++-----
 tests/unittests/test_cli.py | 15 +++++++++++++++
 2 files changed, 19 insertions(+), 5 deletions(-)

(limited to 'tests/unittests/test_cli.py')

diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 5b467979..68563e0c 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -676,11 +676,10 @@ def main_features(name, args):
 
 
 def main(sysv_args=None):
-    if sysv_args is not None:
-        parser = argparse.ArgumentParser(prog=sysv_args[0])
-        sysv_args = sysv_args[1:]
-    else:
-        parser = argparse.ArgumentParser()
+    if not sysv_args:
+        sysv_args = sys.argv
+    parser = argparse.ArgumentParser(prog=sysv_args[0])
+    sysv_args = sysv_args[1:]
 
     # Top level args
     parser.add_argument('--version', '-v', action='version',
diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index 24498802..12f01852 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -70,6 +70,21 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
         self.assertEqual('modules', parseargs.action[0])
         self.assertEqual('main_modules', parseargs.action[1].__name__)
 
+    def test_conditional_subcommands_from_entry_point_sys_argv(self):
+        """Subcommands from entry-point are properly parsed from sys.argv."""
+        expected_errors = [
+            'usage: cloud-init analyze', 'usage: cloud-init devel']
+        conditional_subcommands = ['analyze', 'devel']
+        # The cloud-init entrypoint calls main without passing sys_argv
+        for subcommand in conditional_subcommands:
+            with mock.patch('sys.argv', ['cloud-init', subcommand]):
+                try:
+                    cli.main()
+                except SystemExit as e:
+                    self.assertEqual(2, e.code)  # exit 2 on proper usage docs
+        for error_message in expected_errors:
+            self.assertIn(error_message, self.stderr.getvalue())
+
     def test_analyze_subcommand_parser(self):
         """The subcommand cloud-init analyze calls the correct subparser."""
         self._call_main(['cloud-init', 'analyze'])
-- 
cgit v1.2.3


From a3649e03206a3596131413956ea7ecc18790ec73 Mon Sep 17 00:00:00 2001
From: Lars Kellogg-Stedman <lars@redhat.com>
Date: Tue, 5 Sep 2017 11:03:59 -0600
Subject: relocate tests/unittests/helpers.py to cloudinit/tests

This moves the base test case classes into into cloudinit/tests and
updates all the corresponding imports.
---
 cloudinit/analyze/tests/test_dump.py               |   2 +-
 cloudinit/net/tests/test_dhcp.py                   |   2 +-
 cloudinit/net/tests/test_init.py                   |   2 +-
 cloudinit/tests/helpers.py                         | 395 +++++++++++++++++++++
 cloudinit/tests/test_url_helper.py                 |   2 +-
 tests/unittests/helpers.py                         | 391 --------------------
 tests/unittests/test__init__.py                    |   2 +-
 tests/unittests/test_atomic_helper.py              |   2 +-
 tests/unittests/test_builtin_handlers.py           |   2 +-
 tests/unittests/test_cli.py                        |   2 +-
 tests/unittests/test_cs_util.py                    |   2 +-
 tests/unittests/test_data.py                       |   2 +-
 tests/unittests/test_datasource/test_aliyun.py     |   2 +-
 tests/unittests/test_datasource/test_altcloud.py   |   2 +-
 tests/unittests/test_datasource/test_azure.py      |   6 +-
 .../unittests/test_datasource/test_azure_helper.py |   2 +-
 tests/unittests/test_datasource/test_cloudsigma.py |   2 +-
 tests/unittests/test_datasource/test_cloudstack.py |   2 +-
 tests/unittests/test_datasource/test_common.py     |   2 +-
 .../unittests/test_datasource/test_configdrive.py  |   2 +-
 .../unittests/test_datasource/test_digitalocean.py |   2 +-
 tests/unittests/test_datasource/test_ec2.py        |   2 +-
 tests/unittests/test_datasource/test_gce.py        |   2 +-
 tests/unittests/test_datasource/test_maas.py       |   2 +-
 tests/unittests/test_datasource/test_nocloud.py    |   2 +-
 tests/unittests/test_datasource/test_opennebula.py |   2 +-
 tests/unittests/test_datasource/test_openstack.py  |   2 +-
 tests/unittests/test_datasource/test_ovf.py        |   2 +-
 tests/unittests/test_datasource/test_scaleway.py   |   2 +-
 tests/unittests/test_datasource/test_smartos.py    |   2 +-
 tests/unittests/test_distros/test_arch.py          |   2 +-
 tests/unittests/test_distros/test_create_users.py  |   2 +-
 tests/unittests/test_distros/test_debian.py        |   2 +-
 tests/unittests/test_distros/test_generic.py       |   2 +-
 tests/unittests/test_distros/test_netconfig.py     |   2 +-
 tests/unittests/test_distros/test_opensuse.py      |   2 +-
 tests/unittests/test_distros/test_resolv.py        |   2 +-
 tests/unittests/test_distros/test_sles.py          |   2 +-
 tests/unittests/test_distros/test_sysconfig.py     |   2 +-
 .../test_distros/test_user_data_normalize.py       |   2 +-
 tests/unittests/test_ds_identify.py                |   3 +-
 tests/unittests/test_ec2_util.py                   |   2 +-
 tests/unittests/test_filters/test_launch_index.py  |   2 +-
 .../test_handler/test_handler_apt_conf_v1.py       |   2 +-
 .../test_handler_apt_configure_sources_list_v1.py  |   2 +-
 .../test_handler_apt_configure_sources_list_v3.py  |   2 +-
 .../test_handler/test_handler_apt_source_v1.py     |   2 +-
 .../test_handler/test_handler_apt_source_v3.py     |   2 +-
 .../test_handler/test_handler_ca_certs.py          |   2 +-
 tests/unittests/test_handler/test_handler_chef.py  |   2 +-
 tests/unittests/test_handler/test_handler_debug.py |   2 +-
 .../test_handler/test_handler_disk_setup.py        |   2 +-
 .../test_handler/test_handler_growpart.py          |   2 +-
 .../test_handler/test_handler_landscape.py         |   5 +-
 .../unittests/test_handler/test_handler_locale.py  |   2 +-
 tests/unittests/test_handler/test_handler_lxd.py   |   2 +-
 .../test_handler/test_handler_mcollective.py       |   2 +-
 .../unittests/test_handler/test_handler_mounts.py  |   2 +-
 tests/unittests/test_handler/test_handler_ntp.py   |   2 +-
 .../test_handler/test_handler_power_state.py       |   4 +-
 .../unittests/test_handler/test_handler_puppet.py  |   2 +-
 .../unittests/test_handler/test_handler_rsyslog.py |   2 +-
 .../unittests/test_handler/test_handler_runcmd.py  |   2 +-
 .../test_handler/test_handler_seed_random.py       |   2 +-
 .../test_handler/test_handler_set_hostname.py      |   2 +-
 .../unittests/test_handler/test_handler_snappy.py  |   4 +-
 .../test_handler/test_handler_spacewalk.py         |   2 +-
 .../test_handler/test_handler_timezone.py          |   2 +-
 .../test_handler/test_handler_write_files.py       |   2 +-
 .../test_handler/test_handler_yum_add_repo.py      |   2 +-
 tests/unittests/test_handler/test_schema.py        |   2 +-
 tests/unittests/test_helpers.py                    |   2 +-
 tests/unittests/test_log.py                        |   2 +-
 tests/unittests/test_merging.py                    |   2 +-
 tests/unittests/test_net.py                        |   8 +-
 tests/unittests/test_pathprefix2dict.py            |   2 +-
 tests/unittests/test_registry.py                   |   2 +-
 tests/unittests/test_reporting.py                  |   2 +-
 tests/unittests/test_rh_subscription.py            |   2 +-
 tests/unittests/test_runs/test_merge_run.py        |   2 +-
 tests/unittests/test_runs/test_simple_run.py       |   2 +-
 tests/unittests/test_sshutil.py                    |   3 +-
 tests/unittests/test_templating.py                 |   2 +-
 tests/unittests/test_util.py                       |   2 +-
 tests/unittests/test_version.py                    |   2 +-
 tests/unittests/test_vmware_config_file.py         |   2 +-
 86 files changed, 491 insertions(+), 482 deletions(-)
 create mode 100644 cloudinit/tests/helpers.py
 delete mode 100644 tests/unittests/helpers.py

(limited to 'tests/unittests/test_cli.py')

diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py
index 2c0885d0..f4c42841 100644
--- a/cloudinit/analyze/tests/test_dump.py
+++ b/cloudinit/analyze/tests/test_dump.py
@@ -6,7 +6,7 @@ from textwrap import dedent
 from cloudinit.analyze.dump import (
     dump_events, parse_ci_logline, parse_timestamp)
 from cloudinit.util import subp, write_file
-from tests.unittests.helpers import CiTestCase
+from cloudinit.tests.helpers import CiTestCase
 
 
 class TestParseTimestamp(CiTestCase):
diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py
index 47d8d461..4a37e98a 100644
--- a/cloudinit/net/tests/test_dhcp.py
+++ b/cloudinit/net/tests/test_dhcp.py
@@ -8,7 +8,7 @@ from cloudinit.net.dhcp import (
     InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery,
     parse_dhcp_lease_file, dhcp_discovery)
 from cloudinit.util import ensure_file, write_file
-from tests.unittests.helpers import CiTestCase
+from cloudinit.tests.helpers import CiTestCase
 
 
 class TestParseDHCPLeasesFile(CiTestCase):
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index cc052a7d..8cb4114e 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -7,7 +7,7 @@ import os
 
 import cloudinit.net as net
 from cloudinit.util import ensure_file, write_file, ProcessExecutionError
-from tests.unittests.helpers import CiTestCase
+from cloudinit.tests.helpers import CiTestCase
 
 
 class TestSysDevPath(CiTestCase):
diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
new file mode 100644
index 00000000..28e26622
--- /dev/null
+++ b/cloudinit/tests/helpers.py
@@ -0,0 +1,395 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from __future__ import print_function
+
+import functools
+import json
+import logging
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+import mock
+import six
+import unittest2
+
+try:
+    from contextlib import ExitStack
+except ImportError:
+    from contextlib2 import ExitStack
+
+from cloudinit import helpers as ch
+from cloudinit import util
+
+# Used for skipping tests
+SkipTest = unittest2.SkipTest
+
+# Used for detecting different python versions
+PY2 = False
+PY26 = False
+PY27 = False
+PY3 = False
+
+_PY_VER = sys.version_info
+_PY_MAJOR, _PY_MINOR, _PY_MICRO = _PY_VER[0:3]
+if (_PY_MAJOR, _PY_MINOR) <= (2, 6):
+    if (_PY_MAJOR, _PY_MINOR) == (2, 6):
+        PY26 = True
+    if (_PY_MAJOR, _PY_MINOR) >= (2, 0):
+        PY2 = True
+else:
+    if (_PY_MAJOR, _PY_MINOR) == (2, 7):
+        PY27 = True
+        PY2 = True
+    if (_PY_MAJOR, _PY_MINOR) >= (3, 0):
+        PY3 = True
+
+
+# Makes the old path start
+# with new base instead of whatever
+# it previously had
+def rebase_path(old_path, new_base):
+    if old_path.startswith(new_base):
+        # Already handled...
+        return old_path
+    # Retarget the base of that path
+    # to the new base instead of the
+    # old one...
+    path = os.path.join(new_base, old_path.lstrip("/"))
+    path = os.path.abspath(path)
+    return path
+
+
+# Can work on anything that takes a path as arguments
+def retarget_many_wrapper(new_base, am, old_func):
+    def wrapper(*args, **kwds):
+        n_args = list(args)
+        nam = am
+        if am == -1:
+            nam = len(n_args)
+        for i in range(0, nam):
+            path = args[i]
+            # patchOS() wraps various os and os.path functions, however in
+            # Python 3 some of these now accept file-descriptors (integers).
+            # That breaks rebase_path() so in lieu of a better solution, just
+            # don't rebase if we get a fd.
+            if isinstance(path, six.string_types):
+                n_args[i] = rebase_path(path, new_base)
+        return old_func(*n_args, **kwds)
+    return wrapper
+
+
+class TestCase(unittest2.TestCase):
+
+    def reset_global_state(self):
+        """Reset any global state to its original settings.
+
+        cloudinit caches some values in cloudinit.util.  Unit tests that
+        involved those cached paths were then subject to failure if the order
+        of invocation changed (LP: #1703697).
+
+        This function resets any of these global state variables to their
+        initial state.
+
+        In the future this should really be done with some registry that
+        can then be cleaned in a more obvious way.
+        """
+        util.PROC_CMDLINE = None
+        util._DNS_REDIRECT_IP = None
+        util._LSB_RELEASE = {}
+
+    def setUp(self):
+        super(TestCase, self).setUp()
+        self.reset_global_state()
+
+
+class CiTestCase(TestCase):
+    """This is the preferred test case base class unless user
+       needs other test case classes below."""
+
+    # Subclass overrides for specific test behavior
+    # Whether or not a unit test needs logfile setup
+    with_logs = False
+
+    def setUp(self):
+        super(CiTestCase, self).setUp()
+        if self.with_logs:
+            # Create a log handler so unit tests can search expected logs.
+            self.logger = logging.getLogger()
+            self.logs = six.StringIO()
+            formatter = logging.Formatter('%(levelname)s: %(message)s')
+            handler = logging.StreamHandler(self.logs)
+            handler.setFormatter(formatter)
+            self.old_handlers = self.logger.handlers
+            self.logger.handlers = [handler]
+
+    def tearDown(self):
+        if self.with_logs:
+            # Remove the handler we setup
+            logging.getLogger().handlers = self.old_handlers
+        super(CiTestCase, self).tearDown()
+
+    def tmp_dir(self, dir=None, cleanup=True):
+        # return a full path to a temporary directory that will be cleaned up.
+        if dir is None:
+            tmpd = tempfile.mkdtemp(
+                prefix="ci-%s." % self.__class__.__name__)
+        else:
+            tmpd = tempfile.mkdtemp(dir=dir)
+        self.addCleanup(functools.partial(shutil.rmtree, tmpd))
+        return tmpd
+
+    def tmp_path(self, path, dir=None):
+        # return an absolute path to 'path' under dir.
+        # if dir is None, one will be created with tmp_dir()
+        # the file is not created or modified.
+        if dir is None:
+            dir = self.tmp_dir()
+        return os.path.normpath(os.path.abspath(os.path.join(dir, path)))
+
+
+class ResourceUsingTestCase(CiTestCase):
+
+    def setUp(self):
+        super(ResourceUsingTestCase, self).setUp()
+        self.resource_path = None
+
+    def resourceLocation(self, subname=None):
+        if self.resource_path is None:
+            paths = [
+                os.path.join('tests', 'data'),
+                os.path.join('data'),
+                os.path.join(os.pardir, 'tests', 'data'),
+                os.path.join(os.pardir, 'data'),
+            ]
+            for p in paths:
+                if os.path.isdir(p):
+                    self.resource_path = p
+                    break
+        self.assertTrue((self.resource_path and
+                         os.path.isdir(self.resource_path)),
+                        msg="Unable to locate test resource data path!")
+        if not subname:
+            return self.resource_path
+        return os.path.join(self.resource_path, subname)
+
+    def readResource(self, name):
+        where = self.resourceLocation(name)
+        with open(where, 'r') as fh:
+            return fh.read()
+
+    def getCloudPaths(self, ds=None):
+        tmpdir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, tmpdir)
+        cp = ch.Paths({'cloud_dir': tmpdir,
+                       'templates_dir': self.resourceLocation()},
+                      ds=ds)
+        return cp
+
+
+class FilesystemMockingTestCase(ResourceUsingTestCase):
+
+    def setUp(self):
+        super(FilesystemMockingTestCase, self).setUp()
+        self.patched_funcs = ExitStack()
+
+    def tearDown(self):
+        self.patched_funcs.close()
+        ResourceUsingTestCase.tearDown(self)
+
+    def replicateTestRoot(self, example_root, target_root):
+        real_root = self.resourceLocation()
+        real_root = os.path.join(real_root, 'roots', example_root)
+        for (dir_path, _dirnames, filenames) in os.walk(real_root):
+            real_path = dir_path
+            make_path = rebase_path(real_path[len(real_root):], target_root)
+            util.ensure_dir(make_path)
+            for f in filenames:
+                real_path = util.abs_join(real_path, f)
+                make_path = util.abs_join(make_path, f)
+                shutil.copy(real_path, make_path)
+
+    def patchUtils(self, new_root):
+        patch_funcs = {
+            util: [('write_file', 1),
+                   ('append_file', 1),
+                   ('load_file', 1),
+                   ('ensure_dir', 1),
+                   ('chmod', 1),
+                   ('delete_dir_contents', 1),
+                   ('del_file', 1),
+                   ('sym_link', -1),
+                   ('copy', -1)],
+        }
+        for (mod, funcs) in patch_funcs.items():
+            for (f, am) in funcs:
+                func = getattr(mod, f)
+                trap_func = retarget_many_wrapper(new_root, am, func)
+                self.patched_funcs.enter_context(
+                    mock.patch.object(mod, f, trap_func))
+
+        # Handle subprocess calls
+        func = getattr(util, 'subp')
+
+        def nsubp(*_args, **_kwargs):
+            return ('', '')
+
+        self.patched_funcs.enter_context(
+            mock.patch.object(util, 'subp', nsubp))
+
+        def null_func(*_args, **_kwargs):
+            return None
+
+        for f in ['chownbyid', 'chownbyname']:
+            self.patched_funcs.enter_context(
+                mock.patch.object(util, f, null_func))
+
+    def patchOS(self, new_root):
+        patch_funcs = {
+            os.path: [('isfile', 1), ('exists', 1),
+                      ('islink', 1), ('isdir', 1)],
+            os: [('listdir', 1), ('mkdir', 1),
+                 ('lstat', 1), ('symlink', 2)],
+        }
+        for (mod, funcs) in patch_funcs.items():
+            for f, nargs in funcs:
+                func = getattr(mod, f)
+                trap_func = retarget_many_wrapper(new_root, nargs, func)
+                self.patched_funcs.enter_context(
+                    mock.patch.object(mod, f, trap_func))
+
+    def patchOpen(self, new_root):
+        trap_func = retarget_many_wrapper(new_root, 1, open)
+        name = 'builtins.open' if PY3 else '__builtin__.open'
+        self.patched_funcs.enter_context(mock.patch(name, trap_func))
+
+    def patchStdoutAndStderr(self, stdout=None, stderr=None):
+        if stdout is not None:
+            self.patched_funcs.enter_context(
+                mock.patch.object(sys, 'stdout', stdout))
+        if stderr is not None:
+            self.patched_funcs.enter_context(
+                mock.patch.object(sys, 'stderr', stderr))
+
+    def reRoot(self, root=None):
+        if root is None:
+            root = self.tmp_dir()
+        self.patchUtils(root)
+        self.patchOS(root)
+        return root
+
+
+class HttprettyTestCase(CiTestCase):
+    # necessary as http_proxy gets in the way of httpretty
+    # https://github.com/gabrielfalcao/HTTPretty/issues/122
+
+    def setUp(self):
+        self.restore_proxy = os.environ.get('http_proxy')
+        if self.restore_proxy is not None:
+            del os.environ['http_proxy']
+        super(HttprettyTestCase, self).setUp()
+
+    def tearDown(self):
+        if self.restore_proxy:
+            os.environ['http_proxy'] = self.restore_proxy
+        super(HttprettyTestCase, self).tearDown()
+
+
+def populate_dir(path, files):
+    if not os.path.exists(path):
+        os.makedirs(path)
+    ret = []
+    for (name, content) in files.items():
+        p = os.path.sep.join([path, name])
+        util.ensure_dir(os.path.dirname(p))
+        with open(p, "wb") as fp:
+            if isinstance(content, six.binary_type):
+                fp.write(content)
+            else:
+                fp.write(content.encode('utf-8'))
+            fp.close()
+        ret.append(p)
+
+    return ret
+
+
+def dir2dict(startdir, prefix=None):
+    flist = {}
+    if prefix is None:
+        prefix = startdir
+    for root, dirs, files in os.walk(startdir):
+        for fname in files:
+            fpath = os.path.join(root, fname)
+            key = fpath[len(prefix):]
+            flist[key] = util.load_file(fpath)
+    return flist
+
+
+def json_dumps(data):
+    # print data in nicely formatted json.
+    return json.dumps(data, indent=1, sort_keys=True,
+                      separators=(',', ': '))
+
+
+def wrap_and_call(prefix, mocks, func, *args, **kwargs):
+    """
+    call func(args, **kwargs) with mocks applied, then unapplies mocks
+    nicer to read than repeating dectorators on each function
+
+    prefix: prefix for mock names (e.g. 'cloudinit.stages.util') or None
+    mocks: dictionary of names (under 'prefix') to mock and either
+        a return value or a dictionary to pass to the mock.patch call
+    func: function to call with mocks applied
+    *args,**kwargs: arguments for 'func'
+
+    return_value: return from 'func'
+    """
+    delim = '.'
+    if prefix is None:
+        prefix = ''
+    prefix = prefix.rstrip(delim)
+    unwraps = []
+    for fname, kw in mocks.items():
+        if prefix:
+            fname = delim.join((prefix, fname))
+        if not isinstance(kw, dict):
+            kw = {'return_value': kw}
+        p = mock.patch(fname, **kw)
+        p.start()
+        unwraps.append(p)
+    try:
+        return func(*args, **kwargs)
+    finally:
+        for p in unwraps:
+            p.stop()
+
+
+try:
+    skipIf = unittest.skipIf
+except AttributeError:
+    # Python 2.6.  Doesn't have to be high fidelity.
+    def skipIf(condition, reason):
+        def decorator(func):
+            def wrapper(*args, **kws):
+                if condition:
+                    return func(*args, **kws)
+                else:
+                    print(reason, file=sys.stderr)
+            return wrapper
+        return decorator
+
+
+# older versions of mock do not have the useful 'assert_not_called'
+if not hasattr(mock.Mock, 'assert_not_called'):
+    def __mock_assert_not_called(mmock):
+        if mmock.call_count != 0:
+            msg = ("[citest] Expected '%s' to not have been called. "
+                   "Called %s times." %
+                   (mmock._mock_name or 'mock', mmock.call_count))
+            raise AssertionError(msg)
+    mock.Mock.assert_not_called = __mock_assert_not_called
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py
index 349110d9..b778a3a7 100644
--- a/cloudinit/tests/test_url_helper.py
+++ b/cloudinit/tests/test_url_helper.py
@@ -1,7 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit.url_helper import oauth_headers
-from tests.unittests.helpers import CiTestCase, mock, skipIf
+from cloudinit.tests.helpers import CiTestCase, mock, skipIf
 
 
 try:
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
deleted file mode 100644
index bf1dc5df..00000000
--- a/tests/unittests/helpers.py
+++ /dev/null
@@ -1,391 +0,0 @@
-# This file is part of cloud-init. See LICENSE file for license information.
-
-from __future__ import print_function
-
-import functools
-import json
-import logging
-import os
-import shutil
-import sys
-import tempfile
-import unittest
-
-import mock
-import six
-import unittest2
-
-try:
-    from contextlib import ExitStack
-except ImportError:
-    from contextlib2 import ExitStack
-
-from cloudinit import helpers as ch
-from cloudinit import util
-
-# Used for skipping tests
-SkipTest = unittest2.SkipTest
-
-# Used for detecting different python versions
-PY2 = False
-PY26 = False
-PY27 = False
-PY3 = False
-
-_PY_VER = sys.version_info
-_PY_MAJOR, _PY_MINOR, _PY_MICRO = _PY_VER[0:3]
-if (_PY_MAJOR, _PY_MINOR) <= (2, 6):
-    if (_PY_MAJOR, _PY_MINOR) == (2, 6):
-        PY26 = True
-    if (_PY_MAJOR, _PY_MINOR) >= (2, 0):
-        PY2 = True
-else:
-    if (_PY_MAJOR, _PY_MINOR) == (2, 7):
-        PY27 = True
-        PY2 = True
-    if (_PY_MAJOR, _PY_MINOR) >= (3, 0):
-        PY3 = True
-
-
-# Makes the old path start
-# with new base instead of whatever
-# it previously had
-def rebase_path(old_path, new_base):
-    if old_path.startswith(new_base):
-        # Already handled...
-        return old_path
-    # Retarget the base of that path
-    # to the new base instead of the
-    # old one...
-    path = os.path.join(new_base, old_path.lstrip("/"))
-    path = os.path.abspath(path)
-    return path
-
-
-# Can work on anything that takes a path as arguments
-def retarget_many_wrapper(new_base, am, old_func):
-    def wrapper(*args, **kwds):
-        n_args = list(args)
-        nam = am
-        if am == -1:
-            nam = len(n_args)
-        for i in range(0, nam):
-            path = args[i]
-            # patchOS() wraps various os and os.path functions, however in
-            # Python 3 some of these now accept file-descriptors (integers).
-            # That breaks rebase_path() so in lieu of a better solution, just
-            # don't rebase if we get a fd.
-            if isinstance(path, six.string_types):
-                n_args[i] = rebase_path(path, new_base)
-        return old_func(*n_args, **kwds)
-    return wrapper
-
-
-class TestCase(unittest2.TestCase):
-    def reset_global_state(self):
-        """Reset any global state to its original settings.
-
-        cloudinit caches some values in cloudinit.util.  Unit tests that
-        involved those cached paths were then subject to failure if the order
-        of invocation changed (LP: #1703697).
-
-        This function resets any of these global state variables to their
-        initial state.
-
-        In the future this should really be done with some registry that
-        can then be cleaned in a more obvious way.
-        """
-        util.PROC_CMDLINE = None
-        util._DNS_REDIRECT_IP = None
-        util._LSB_RELEASE = {}
-
-    def setUp(self):
-        super(unittest2.TestCase, self).setUp()
-        self.reset_global_state()
-
-
-class CiTestCase(TestCase):
-    """This is the preferred test case base class unless user
-       needs other test case classes below."""
-
-    # Subclass overrides for specific test behavior
-    # Whether or not a unit test needs logfile setup
-    with_logs = False
-
-    def setUp(self):
-        super(CiTestCase, self).setUp()
-        if self.with_logs:
-            # Create a log handler so unit tests can search expected logs.
-            self.logger = logging.getLogger()
-            self.logs = six.StringIO()
-            formatter = logging.Formatter('%(levelname)s: %(message)s')
-            handler = logging.StreamHandler(self.logs)
-            handler.setFormatter(formatter)
-            self.old_handlers = self.logger.handlers
-            self.logger.handlers = [handler]
-
-    def tearDown(self):
-        if self.with_logs:
-            # Remove the handler we setup
-            logging.getLogger().handlers = self.old_handlers
-        super(CiTestCase, self).tearDown()
-
-    def tmp_dir(self, dir=None, cleanup=True):
-        # return a full path to a temporary directory that will be cleaned up.
-        if dir is None:
-            tmpd = tempfile.mkdtemp(
-                prefix="ci-%s." % self.__class__.__name__)
-        else:
-            tmpd = tempfile.mkdtemp(dir=dir)
-        self.addCleanup(functools.partial(shutil.rmtree, tmpd))
-        return tmpd
-
-    def tmp_path(self, path, dir=None):
-        # return an absolute path to 'path' under dir.
-        # if dir is None, one will be created with tmp_dir()
-        # the file is not created or modified.
-        if dir is None:
-            dir = self.tmp_dir()
-        return os.path.normpath(os.path.abspath(os.path.join(dir, path)))
-
-
-class ResourceUsingTestCase(CiTestCase):
-    def setUp(self):
-        super(ResourceUsingTestCase, self).setUp()
-        self.resource_path = None
-
-    def resourceLocation(self, subname=None):
-        if self.resource_path is None:
-            paths = [
-                os.path.join('tests', 'data'),
-                os.path.join('data'),
-                os.path.join(os.pardir, 'tests', 'data'),
-                os.path.join(os.pardir, 'data'),
-            ]
-            for p in paths:
-                if os.path.isdir(p):
-                    self.resource_path = p
-                    break
-        self.assertTrue((self.resource_path and
-                         os.path.isdir(self.resource_path)),
-                        msg="Unable to locate test resource data path!")
-        if not subname:
-            return self.resource_path
-        return os.path.join(self.resource_path, subname)
-
-    def readResource(self, name):
-        where = self.resourceLocation(name)
-        with open(where, 'r') as fh:
-            return fh.read()
-
-    def getCloudPaths(self, ds=None):
-        tmpdir = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, tmpdir)
-        cp = ch.Paths({'cloud_dir': tmpdir,
-                       'templates_dir': self.resourceLocation()},
-                      ds=ds)
-        return cp
-
-
-class FilesystemMockingTestCase(ResourceUsingTestCase):
-    def setUp(self):
-        super(FilesystemMockingTestCase, self).setUp()
-        self.patched_funcs = ExitStack()
-
-    def tearDown(self):
-        self.patched_funcs.close()
-        ResourceUsingTestCase.tearDown(self)
-
-    def replicateTestRoot(self, example_root, target_root):
-        real_root = self.resourceLocation()
-        real_root = os.path.join(real_root, 'roots', example_root)
-        for (dir_path, _dirnames, filenames) in os.walk(real_root):
-            real_path = dir_path
-            make_path = rebase_path(real_path[len(real_root):], target_root)
-            util.ensure_dir(make_path)
-            for f in filenames:
-                real_path = util.abs_join(real_path, f)
-                make_path = util.abs_join(make_path, f)
-                shutil.copy(real_path, make_path)
-
-    def patchUtils(self, new_root):
-        patch_funcs = {
-            util: [('write_file', 1),
-                   ('append_file', 1),
-                   ('load_file', 1),
-                   ('ensure_dir', 1),
-                   ('chmod', 1),
-                   ('delete_dir_contents', 1),
-                   ('del_file', 1),
-                   ('sym_link', -1),
-                   ('copy', -1)],
-        }
-        for (mod, funcs) in patch_funcs.items():
-            for (f, am) in funcs:
-                func = getattr(mod, f)
-                trap_func = retarget_many_wrapper(new_root, am, func)
-                self.patched_funcs.enter_context(
-                    mock.patch.object(mod, f, trap_func))
-
-        # Handle subprocess calls
-        func = getattr(util, 'subp')
-
-        def nsubp(*_args, **_kwargs):
-            return ('', '')
-
-        self.patched_funcs.enter_context(
-            mock.patch.object(util, 'subp', nsubp))
-
-        def null_func(*_args, **_kwargs):
-            return None
-
-        for f in ['chownbyid', 'chownbyname']:
-            self.patched_funcs.enter_context(
-                mock.patch.object(util, f, null_func))
-
-    def patchOS(self, new_root):
-        patch_funcs = {
-            os.path: [('isfile', 1), ('exists', 1),
-                      ('islink', 1), ('isdir', 1)],
-            os: [('listdir', 1), ('mkdir', 1),
-                 ('lstat', 1), ('symlink', 2)],
-        }
-        for (mod, funcs) in patch_funcs.items():
-            for f, nargs in funcs:
-                func = getattr(mod, f)
-                trap_func = retarget_many_wrapper(new_root, nargs, func)
-                self.patched_funcs.enter_context(
-                    mock.patch.object(mod, f, trap_func))
-
-    def patchOpen(self, new_root):
-        trap_func = retarget_many_wrapper(new_root, 1, open)
-        name = 'builtins.open' if PY3 else '__builtin__.open'
-        self.patched_funcs.enter_context(mock.patch(name, trap_func))
-
-    def patchStdoutAndStderr(self, stdout=None, stderr=None):
-        if stdout is not None:
-            self.patched_funcs.enter_context(
-                mock.patch.object(sys, 'stdout', stdout))
-        if stderr is not None:
-            self.patched_funcs.enter_context(
-                mock.patch.object(sys, 'stderr', stderr))
-
-    def reRoot(self, root=None):
-        if root is None:
-            root = self.tmp_dir()
-        self.patchUtils(root)
-        self.patchOS(root)
-        return root
-
-
-class HttprettyTestCase(CiTestCase):
-    # necessary as http_proxy gets in the way of httpretty
-    # https://github.com/gabrielfalcao/HTTPretty/issues/122
-    def setUp(self):
-        self.restore_proxy = os.environ.get('http_proxy')
-        if self.restore_proxy is not None:
-            del os.environ['http_proxy']
-        super(HttprettyTestCase, self).setUp()
-
-    def tearDown(self):
-        if self.restore_proxy:
-            os.environ['http_proxy'] = self.restore_proxy
-        super(HttprettyTestCase, self).tearDown()
-
-
-def populate_dir(path, files):
-    if not os.path.exists(path):
-        os.makedirs(path)
-    ret = []
-    for (name, content) in files.items():
-        p = os.path.sep.join([path, name])
-        util.ensure_dir(os.path.dirname(p))
-        with open(p, "wb") as fp:
-            if isinstance(content, six.binary_type):
-                fp.write(content)
-            else:
-                fp.write(content.encode('utf-8'))
-            fp.close()
-        ret.append(p)
-
-    return ret
-
-
-def dir2dict(startdir, prefix=None):
-    flist = {}
-    if prefix is None:
-        prefix = startdir
-    for root, dirs, files in os.walk(startdir):
-        for fname in files:
-            fpath = os.path.join(root, fname)
-            key = fpath[len(prefix):]
-            flist[key] = util.load_file(fpath)
-    return flist
-
-
-def json_dumps(data):
-    # print data in nicely formatted json.
-    return json.dumps(data, indent=1, sort_keys=True,
-                      separators=(',', ': '))
-
-
-def wrap_and_call(prefix, mocks, func, *args, **kwargs):
-    """
-    call func(args, **kwargs) with mocks applied, then unapplies mocks
-    nicer to read than repeating dectorators on each function
-
-    prefix: prefix for mock names (e.g. 'cloudinit.stages.util') or None
-    mocks: dictionary of names (under 'prefix') to mock and either
-        a return value or a dictionary to pass to the mock.patch call
-    func: function to call with mocks applied
-    *args,**kwargs: arguments for 'func'
-
-    return_value: return from 'func'
-    """
-    delim = '.'
-    if prefix is None:
-        prefix = ''
-    prefix = prefix.rstrip(delim)
-    unwraps = []
-    for fname, kw in mocks.items():
-        if prefix:
-            fname = delim.join((prefix, fname))
-        if not isinstance(kw, dict):
-            kw = {'return_value': kw}
-        p = mock.patch(fname, **kw)
-        p.start()
-        unwraps.append(p)
-    try:
-        return func(*args, **kwargs)
-    finally:
-        for p in unwraps:
-            p.stop()
-
-
-try:
-    skipIf = unittest.skipIf
-except AttributeError:
-    # Python 2.6.  Doesn't have to be high fidelity.
-    def skipIf(condition, reason):
-        def decorator(func):
-            def wrapper(*args, **kws):
-                if condition:
-                    return func(*args, **kws)
-                else:
-                    print(reason, file=sys.stderr)
-            return wrapper
-        return decorator
-
-
-# older versions of mock do not have the useful 'assert_not_called'
-if not hasattr(mock.Mock, 'assert_not_called'):
-    def __mock_assert_not_called(mmock):
-        if mmock.call_count != 0:
-            msg = ("[citest] Expected '%s' to not have been called. "
-                   "Called %s times." %
-                   (mmock._mock_name or 'mock', mmock.call_count))
-            raise AssertionError(msg)
-    mock.Mock.assert_not_called = __mock_assert_not_called
-
-
-# vi: ts=4 expandtab
diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py
index 781f6d54..25878d7a 100644
--- a/tests/unittests/test__init__.py
+++ b/tests/unittests/test__init__.py
@@ -12,7 +12,7 @@ from cloudinit import settings
 from cloudinit import url_helper
 from cloudinit import util
 
-from .helpers import TestCase, CiTestCase, ExitStack, mock
+from cloudinit.tests.helpers import TestCase, CiTestCase, ExitStack, mock
 
 
 class FakeModule(handlers.Handler):
diff --git a/tests/unittests/test_atomic_helper.py b/tests/unittests/test_atomic_helper.py
index 515919d8..0101b0e3 100644
--- a/tests/unittests/test_atomic_helper.py
+++ b/tests/unittests/test_atomic_helper.py
@@ -6,7 +6,7 @@ import stat
 
 from cloudinit import atomic_helper
 
-from .helpers import CiTestCase
+from cloudinit.tests.helpers import CiTestCase
 
 
 class TestAtomicHelper(CiTestCase):
diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py
index dd9d0357..9751ed95 100644
--- a/tests/unittests/test_builtin_handlers.py
+++ b/tests/unittests/test_builtin_handlers.py
@@ -11,7 +11,7 @@ try:
 except ImportError:
     import mock
 
-from . import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 from cloudinit import handlers
 from cloudinit import helpers
diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index 12f01852..495bdc9f 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -2,7 +2,7 @@
 
 import six
 
-from . import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 from cloudinit.cmd import main as cli
 
diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py
index b8f5031c..ee88520d 100644
--- a/tests/unittests/test_cs_util.py
+++ b/tests/unittests/test_cs_util.py
@@ -2,7 +2,7 @@
 
 from __future__ import print_function
 
-from . import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 from cloudinit.cs_utils import Cepko
 
diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py
index 4ad86bb6..6d621d26 100644
--- a/tests/unittests/test_data.py
+++ b/tests/unittests/test_data.py
@@ -27,7 +27,7 @@ from cloudinit import stages
 from cloudinit import user_data as ud
 from cloudinit import util
 
-from . import helpers
+from cloudinit.tests import helpers
 
 
 INSTANCE_ID = "i-testing"
diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py
index 996560e4..82ee9714 100644
--- a/tests/unittests/test_datasource/test_aliyun.py
+++ b/tests/unittests/test_datasource/test_aliyun.py
@@ -5,9 +5,9 @@ import httpretty
 import mock
 import os
 
-from .. import helpers as test_helpers
 from cloudinit import helpers
 from cloudinit.sources import DataSourceAliYun as ay
+from cloudinit.tests import helpers as test_helpers
 
 DEFAULT_METADATA = {
     'instance-id': 'aliyun-test-vm-00',
diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py
index 9c46abc1..3b274d90 100644
--- a/tests/unittests/test_datasource/test_altcloud.py
+++ b/tests/unittests/test_datasource/test_altcloud.py
@@ -18,7 +18,7 @@ import tempfile
 from cloudinit import helpers
 from cloudinit import util
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 import cloudinit.sources.DataSourceAltCloud as dsac
 
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 20e70fb7..0a117771 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -6,8 +6,8 @@ from cloudinit.sources import DataSourceAzure as dsaz
 from cloudinit.util import find_freebsd_part
 from cloudinit.util import get_path_dev_freebsd
 
-from ..helpers import (CiTestCase, TestCase, populate_dir, mock,
-                       ExitStack, PY26, SkipTest)
+from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock,
+                                     ExitStack, PY26, SkipTest)
 
 import crypt
 import os
@@ -871,6 +871,7 @@ class TestLoadAzureDsDir(CiTestCase):
 
 
 class TestReadAzureOvf(TestCase):
+
     def test_invalid_xml_raises_non_azure_ds(self):
         invalid_xml = "<foo>" + construct_valid_ovf_env(data={})
         self.assertRaises(dsaz.BrokenAzureDataSource,
@@ -1079,6 +1080,7 @@ class TestCanDevBeReformatted(CiTestCase):
 
 
 class TestAzureNetExists(CiTestCase):
+
     def test_azure_net_must_exist_for_legacy_objpkl(self):
         """DataSourceAzureNet must exist for old obj.pkl files
            that reference it."""
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index b2d2971b..80ce003d 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -3,7 +3,7 @@
 import os
 
 from cloudinit.sources.helpers import azure as azure_helper
-from ..helpers import ExitStack, mock, TestCase
+from cloudinit.tests.helpers import ExitStack, mock, TestCase
 
 
 GOAL_STATE_TEMPLATE = """\
diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py
index 5997102c..e4c59907 100644
--- a/tests/unittests/test_datasource/test_cloudsigma.py
+++ b/tests/unittests/test_datasource/test_cloudsigma.py
@@ -6,7 +6,7 @@ from cloudinit.cs_utils import Cepko
 from cloudinit import sources
 from cloudinit.sources import DataSourceCloudSigma
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 SERVER_CONTEXT = {
     "cpu": 1000,
diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py
index e94aad61..2dc90305 100644
--- a/tests/unittests/test_datasource/test_cloudstack.py
+++ b/tests/unittests/test_datasource/test_cloudstack.py
@@ -3,7 +3,7 @@
 from cloudinit import helpers
 from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack
 
-from ..helpers import TestCase, mock, ExitStack
+from cloudinit.tests.helpers import TestCase, mock, ExitStack
 
 
 class TestCloudStackPasswordFetching(TestCase):
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 4802f105..80b9c650 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -24,7 +24,7 @@ from cloudinit.sources import (
 )
 from cloudinit.sources import DataSourceNone as DSNone
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 DEFAULT_LOCAL = [
     Azure.DataSourceAzure,
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index 337be667..237c189b 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -15,7 +15,7 @@ from cloudinit.sources import DataSourceConfigDrive as ds
 from cloudinit.sources.helpers import openstack
 from cloudinit import util
 
-from ..helpers import TestCase, ExitStack, mock
+from cloudinit.tests.helpers import TestCase, ExitStack, mock
 
 
 PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n'
diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py
index e97a679a..f264f361 100644
--- a/tests/unittests/test_datasource/test_digitalocean.py
+++ b/tests/unittests/test_datasource/test_digitalocean.py
@@ -13,7 +13,7 @@ from cloudinit import settings
 from cloudinit.sources import DataSourceDigitalOcean
 from cloudinit.sources.helpers import digitalocean
 
-from ..helpers import mock, TestCase
+from cloudinit.tests.helpers import mock, TestCase
 
 DO_MULTIPLE_KEYS = ["ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@do.co",
                     "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@do.co"]
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index b7a84e21..9fb90483 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -4,9 +4,9 @@ import copy
 import httpretty
 import mock
 
-from .. import helpers as test_helpers
 from cloudinit import helpers
 from cloudinit.sources import DataSourceEc2 as ec2
+from cloudinit.tests import helpers as test_helpers
 
 
 # collected from api version 2016-09-02/ with
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index ad608bec..50e49a10 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -15,7 +15,7 @@ from cloudinit import helpers
 from cloudinit import settings
 from cloudinit.sources import DataSourceGCE
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 
 GCE_META = {
diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py
index c1911bf4..289c6a40 100644
--- a/tests/unittests/test_datasource/test_maas.py
+++ b/tests/unittests/test_datasource/test_maas.py
@@ -8,7 +8,7 @@ import yaml
 
 from cloudinit.sources import DataSourceMAAS
 from cloudinit import url_helper
-from ..helpers import TestCase, populate_dir
+from cloudinit.tests.helpers import TestCase, populate_dir
 
 try:
     from unittest import mock
diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py
index ff294395..fea9156b 100644
--- a/tests/unittests/test_datasource/test_nocloud.py
+++ b/tests/unittests/test_datasource/test_nocloud.py
@@ -3,7 +3,7 @@
 from cloudinit import helpers
 from cloudinit.sources import DataSourceNoCloud
 from cloudinit import util
-from ..helpers import TestCase, populate_dir, mock, ExitStack
+from cloudinit.tests.helpers import TestCase, populate_dir, mock, ExitStack
 
 import os
 import shutil
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index b0f8e435..e7d55692 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -3,7 +3,7 @@
 from cloudinit import helpers
 from cloudinit.sources import DataSourceOpenNebula as ds
 from cloudinit import util
-from ..helpers import mock, populate_dir, TestCase
+from cloudinit.tests.helpers import mock, populate_dir, TestCase
 
 import os
 import pwd
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index c2905d1a..177e9808 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -9,7 +9,7 @@ import httpretty as hp
 import json
 import re
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 from six.moves.urllib.parse import urlparse
 from six import StringIO
diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py
index 477cf8ed..9dbf4dd9 100644
--- a/tests/unittests/test_datasource/test_ovf.py
+++ b/tests/unittests/test_datasource/test_ovf.py
@@ -6,7 +6,7 @@
 
 import base64
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 from cloudinit.sources import DataSourceOVF as dsovf
 
diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py
index 65d83ad7..436df9ee 100644
--- a/tests/unittests/test_datasource/test_scaleway.py
+++ b/tests/unittests/test_datasource/test_scaleway.py
@@ -9,7 +9,7 @@ from cloudinit import helpers
 from cloudinit import settings
 from cloudinit.sources import DataSourceScaleway
 
-from ..helpers import mock, HttprettyTestCase, TestCase
+from cloudinit.tests.helpers import mock, HttprettyTestCase, TestCase
 
 
 class DataResponses(object):
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index e3c99bbb..933d5b63 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -33,7 +33,7 @@ import six
 from cloudinit import helpers as c_helpers
 from cloudinit.util import b64e
 
-from ..helpers import mock, FilesystemMockingTestCase, TestCase
+from cloudinit.tests.helpers import mock, FilesystemMockingTestCase, TestCase
 
 SDC_NICS = json.loads("""
 [
diff --git a/tests/unittests/test_distros/test_arch.py b/tests/unittests/test_distros/test_arch.py
index 3d4c9a70..a95ba3b5 100644
--- a/tests/unittests/test_distros/test_arch.py
+++ b/tests/unittests/test_distros/test_arch.py
@@ -3,7 +3,7 @@
 from cloudinit.distros.arch import _render_network
 from cloudinit import util
 
-from ..helpers import (CiTestCase, dir2dict)
+from cloudinit.tests.helpers import (CiTestCase, dir2dict)
 
 from . import _get_distro
 
diff --git a/tests/unittests/test_distros/test_create_users.py b/tests/unittests/test_distros/test_create_users.py
index 1d02f7bd..aa13670a 100644
--- a/tests/unittests/test_distros/test_create_users.py
+++ b/tests/unittests/test_distros/test_create_users.py
@@ -1,7 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit import distros
-from ..helpers import (TestCase, mock)
+from cloudinit.tests.helpers import (TestCase, mock)
 
 
 class MyBaseDistro(distros.Distro):
diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py
index 72d3aad6..da16a797 100644
--- a/tests/unittests/test_distros/test_debian.py
+++ b/tests/unittests/test_distros/test_debian.py
@@ -2,7 +2,7 @@
 
 from cloudinit import distros
 from cloudinit import util
-from ..helpers import (FilesystemMockingTestCase, mock)
+from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock)
 
 
 @mock.patch("cloudinit.distros.debian.util.subp")
diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py
index b355a19e..791fe612 100644
--- a/tests/unittests/test_distros/test_generic.py
+++ b/tests/unittests/test_distros/test_generic.py
@@ -3,7 +3,7 @@
 from cloudinit import distros
 from cloudinit import util
 
-from .. import helpers
+from cloudinit.tests import helpers
 
 import os
 import shutil
diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py
index 6d89dba8..c4bd11bc 100644
--- a/tests/unittests/test_distros/test_netconfig.py
+++ b/tests/unittests/test_distros/test_netconfig.py
@@ -12,7 +12,7 @@ try:
 except ImportError:
     from contextlib2 import ExitStack
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 from cloudinit import distros
 from cloudinit.distros.parsers.sys_conf import SysConf
diff --git a/tests/unittests/test_distros/test_opensuse.py b/tests/unittests/test_distros/test_opensuse.py
index bdb1d633..b9bb9b3e 100644
--- a/tests/unittests/test_distros/test_opensuse.py
+++ b/tests/unittests/test_distros/test_opensuse.py
@@ -1,6 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from ..helpers import CiTestCase
+from cloudinit.tests.helpers import CiTestCase
 
 from . import _get_distro
 
diff --git a/tests/unittests/test_distros/test_resolv.py b/tests/unittests/test_distros/test_resolv.py
index 97168cf9..68ea0083 100644
--- a/tests/unittests/test_distros/test_resolv.py
+++ b/tests/unittests/test_distros/test_resolv.py
@@ -3,7 +3,7 @@
 from cloudinit.distros.parsers import resolv_conf
 from cloudinit.distros import rhel_util
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 import re
 import tempfile
diff --git a/tests/unittests/test_distros/test_sles.py b/tests/unittests/test_distros/test_sles.py
index c656aacc..33e3c457 100644
--- a/tests/unittests/test_distros/test_sles.py
+++ b/tests/unittests/test_distros/test_sles.py
@@ -1,6 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from ..helpers import CiTestCase
+from cloudinit.tests.helpers import CiTestCase
 
 from . import _get_distro
 
diff --git a/tests/unittests/test_distros/test_sysconfig.py b/tests/unittests/test_distros/test_sysconfig.py
index 235ecebb..c1d5b693 100644
--- a/tests/unittests/test_distros/test_sysconfig.py
+++ b/tests/unittests/test_distros/test_sysconfig.py
@@ -4,7 +4,7 @@ import re
 
 from cloudinit.distros.parsers.sys_conf import SysConf
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 
 # Lots of good examples @
diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py
index 88746e0a..0fa9cdb5 100644
--- a/tests/unittests/test_distros/test_user_data_normalize.py
+++ b/tests/unittests/test_distros/test_user_data_normalize.py
@@ -5,7 +5,7 @@ from cloudinit.distros import ug_util
 from cloudinit import helpers
 from cloudinit import settings
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 import mock
 
 
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 8ccfe55c..1a81a89e 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -6,7 +6,8 @@ from uuid import uuid4
 
 from cloudinit import safeyaml
 from cloudinit import util
-from .helpers import CiTestCase, dir2dict, json_dumps, populate_dir
+from cloudinit.tests.helpers import (
+    CiTestCase, dir2dict, json_dumps, populate_dir)
 
 UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu "
                "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux")
diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py
index 65fdb519..af78997f 100644
--- a/tests/unittests/test_ec2_util.py
+++ b/tests/unittests/test_ec2_util.py
@@ -2,7 +2,7 @@
 
 import httpretty as hp
 
-from . import helpers
+from cloudinit.tests import helpers
 
 from cloudinit import ec2_utils as eu
 from cloudinit import url_helper as uh
diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py
index 13137f6d..6364d38e 100644
--- a/tests/unittests/test_filters/test_launch_index.py
+++ b/tests/unittests/test_filters/test_launch_index.py
@@ -2,7 +2,7 @@
 
 import copy
 
-from .. import helpers
+from cloudinit.tests import helpers
 
 from six.moves import filterfalse
 
diff --git a/tests/unittests/test_handler/test_handler_apt_conf_v1.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py
index 554277ff..83f962a9 100644
--- a/tests/unittests/test_handler/test_handler_apt_conf_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py
@@ -3,7 +3,7 @@
 from cloudinit.config import cc_apt_configure
 from cloudinit import util
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 import copy
 import os
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
index f53ddbb2..d2b96f0b 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
@@ -24,7 +24,7 @@ from cloudinit.sources import DataSourceNone
 
 from cloudinit.distros.debian import Distro
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 LOG = logging.getLogger(__name__)
 
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py
index 1ca915b4..f7608c28 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py
@@ -24,7 +24,7 @@ from cloudinit.sources import DataSourceNone
 
 from cloudinit.distros.debian import Distro
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 LOG = logging.getLogger(__name__)
 
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py
index 12502d05..3a3f95ca 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py
@@ -20,7 +20,7 @@ from cloudinit.config import cc_apt_configure
 from cloudinit import gpg
 from cloudinit import util
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py
index 292d3f59..7bb1b7c4 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py
@@ -28,7 +28,7 @@ from cloudinit import util
 from cloudinit.config import cc_apt_configure
 from cloudinit.sources import DataSourceNone
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 EXPECTEDKEY = u"""-----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1
diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py
index 7cee2c3f..06e14db0 100644
--- a/tests/unittests/test_handler/test_handler_ca_certs.py
+++ b/tests/unittests/test_handler/test_handler_ca_certs.py
@@ -5,7 +5,7 @@ from cloudinit.config import cc_ca_certs
 from cloudinit import helpers
 from cloudinit import util
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 import logging
 import shutil
diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py
index 6a152ea2..e5785cfd 100644
--- a/tests/unittests/test_handler/test_handler_chef.py
+++ b/tests/unittests/test_handler/test_handler_chef.py
@@ -14,7 +14,7 @@ from cloudinit import helpers
 from cloudinit.sources import DataSourceNone
 from cloudinit import util
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 LOG = logging.getLogger(__name__)
 
diff --git a/tests/unittests/test_handler/test_handler_debug.py b/tests/unittests/test_handler/test_handler_debug.py
index 1873c3e1..787ba350 100644
--- a/tests/unittests/test_handler/test_handler_debug.py
+++ b/tests/unittests/test_handler/test_handler_debug.py
@@ -11,7 +11,7 @@ from cloudinit import util
 
 from cloudinit.sources import DataSourceNone
 
-from ..helpers import (FilesystemMockingTestCase, mock)
+from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock)
 
 import logging
 import shutil
diff --git a/tests/unittests/test_handler/test_handler_disk_setup.py b/tests/unittests/test_handler/test_handler_disk_setup.py
index 8a6d49ed..5afcacaf 100644
--- a/tests/unittests/test_handler/test_handler_disk_setup.py
+++ b/tests/unittests/test_handler/test_handler_disk_setup.py
@@ -3,7 +3,7 @@
 import random
 
 from cloudinit.config import cc_disk_setup
-from ..helpers import CiTestCase, ExitStack, mock, TestCase
+from cloudinit.tests.helpers import CiTestCase, ExitStack, mock, TestCase
 
 
 class TestIsDiskUsed(TestCase):
diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py
index c5fc8c9b..a3e46351 100644
--- a/tests/unittests/test_handler/test_handler_growpart.py
+++ b/tests/unittests/test_handler/test_handler_growpart.py
@@ -4,7 +4,7 @@ from cloudinit import cloud
 from cloudinit.config import cc_growpart
 from cloudinit import util
 
-from ..helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 import errno
 import logging
diff --git a/tests/unittests/test_handler/test_handler_landscape.py b/tests/unittests/test_handler/test_handler_landscape.py
index 7c247fa9..db92a7e2 100644
--- a/tests/unittests/test_handler/test_handler_landscape.py
+++ b/tests/unittests/test_handler/test_handler_landscape.py
@@ -1,9 +1,10 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit.config import cc_landscape
-from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
-from ..helpers import FilesystemMockingTestCase, mock, wrap_and_call
+from cloudinit.sources import DataSourceNone
+from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock,
+                                     wrap_and_call)
 
 from configobj import ConfigObj
 import logging
diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py
index a789db32..e29a06f9 100644
--- a/tests/unittests/test_handler/test_handler_locale.py
+++ b/tests/unittests/test_handler/test_handler_locale.py
@@ -13,7 +13,7 @@ from cloudinit import util
 
 from cloudinit.sources import DataSourceNoCloud
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 from configobj import ConfigObj
 
diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py
index 351226bf..f132a778 100644
--- a/tests/unittests/test_handler/test_handler_lxd.py
+++ b/tests/unittests/test_handler/test_handler_lxd.py
@@ -3,7 +3,7 @@
 from cloudinit.config import cc_lxd
 from cloudinit.sources import DataSourceNoCloud
 from cloudinit import (distros, helpers, cloud)
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 import logging
 
diff --git a/tests/unittests/test_handler/test_handler_mcollective.py b/tests/unittests/test_handler/test_handler_mcollective.py
index 2a9f3823..7eec7352 100644
--- a/tests/unittests/test_handler/test_handler_mcollective.py
+++ b/tests/unittests/test_handler/test_handler_mcollective.py
@@ -4,7 +4,7 @@ from cloudinit import (cloud, distros, helpers, util)
 from cloudinit.config import cc_mcollective
 from cloudinit.sources import DataSourceNoCloud
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 import configobj
 import logging
diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py
index 650ca0ec..fe492d4b 100644
--- a/tests/unittests/test_handler/test_handler_mounts.py
+++ b/tests/unittests/test_handler/test_handler_mounts.py
@@ -6,7 +6,7 @@ import tempfile
 
 from cloudinit.config import cc_mounts
 
-from .. import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 try:
     from unittest import mock
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 83d5faa2..4f291248 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -3,7 +3,7 @@
 from cloudinit.config import cc_ntp
 from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
-from ..helpers import FilesystemMockingTestCase, mock, skipIf
+from cloudinit.tests.helpers import FilesystemMockingTestCase, mock, skipIf
 
 
 import os
diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py
index e382210d..85a0fe0a 100644
--- a/tests/unittests/test_handler/test_handler_power_state.py
+++ b/tests/unittests/test_handler/test_handler_power_state.py
@@ -4,8 +4,8 @@ import sys
 
 from cloudinit.config import cc_power_state_change as psc
 
-from .. import helpers as t_help
-from ..helpers import mock
+from cloudinit.tests import helpers as t_help
+from cloudinit.tests.helpers import mock
 
 
 class TestLoadPowerState(t_help.TestCase):
diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py
index 805c76ba..0b6e3b58 100644
--- a/tests/unittests/test_handler/test_handler_puppet.py
+++ b/tests/unittests/test_handler/test_handler_puppet.py
@@ -3,7 +3,7 @@
 from cloudinit.config import cc_puppet
 from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
-from ..helpers import CiTestCase, mock
+from cloudinit.tests.helpers import CiTestCase, mock
 
 import logging
 
diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py
index cca06678..8c8e2838 100644
--- a/tests/unittests/test_handler/test_handler_rsyslog.py
+++ b/tests/unittests/test_handler/test_handler_rsyslog.py
@@ -9,7 +9,7 @@ from cloudinit.config.cc_rsyslog import (
     parse_remotes_line, remotes_to_rsyslog_cfg)
 from cloudinit import util
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 
 class TestLoadConfig(t_help.TestCase):
diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py
index 7880ee72..374c1d31 100644
--- a/tests/unittests/test_handler/test_handler_runcmd.py
+++ b/tests/unittests/test_handler/test_handler_runcmd.py
@@ -3,7 +3,7 @@
 from cloudinit.config import cc_runcmd
 from cloudinit.sources import DataSourceNone
 from cloudinit import (distros, helpers, cloud, util)
-from ..helpers import FilesystemMockingTestCase, skipIf
+from cloudinit.tests.helpers import FilesystemMockingTestCase, skipIf
 
 import logging
 import os
diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py
index e5e607fb..f60dedc2 100644
--- a/tests/unittests/test_handler/test_handler_seed_random.py
+++ b/tests/unittests/test_handler/test_handler_seed_random.py
@@ -22,7 +22,7 @@ from cloudinit import util
 
 from cloudinit.sources import DataSourceNone
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 import logging
 
diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py
index 8165bf9a..abdc17e7 100644
--- a/tests/unittests/test_handler/test_handler_set_hostname.py
+++ b/tests/unittests/test_handler/test_handler_set_hostname.py
@@ -7,7 +7,7 @@ from cloudinit import distros
 from cloudinit import helpers
 from cloudinit import util
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 from configobj import ConfigObj
 import logging
diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py
index e4d07622..76b79c29 100644
--- a/tests/unittests/test_handler/test_handler_snappy.py
+++ b/tests/unittests/test_handler/test_handler_snappy.py
@@ -7,9 +7,9 @@ from cloudinit.config.cc_snap_config import (
 from cloudinit import (distros, helpers, cloud, util)
 from cloudinit.config.cc_snap_config import handle as snap_handle
 from cloudinit.sources import DataSourceNone
-from ..helpers import FilesystemMockingTestCase, mock
+from cloudinit.tests.helpers import FilesystemMockingTestCase, mock
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 import logging
 import os
diff --git a/tests/unittests/test_handler/test_handler_spacewalk.py b/tests/unittests/test_handler/test_handler_spacewalk.py
index 28b5892a..ddbf4a79 100644
--- a/tests/unittests/test_handler/test_handler_spacewalk.py
+++ b/tests/unittests/test_handler/test_handler_spacewalk.py
@@ -3,7 +3,7 @@
 from cloudinit.config import cc_spacewalk
 from cloudinit import util
 
-from .. import helpers
+from cloudinit.tests import helpers
 
 import logging
 
diff --git a/tests/unittests/test_handler/test_handler_timezone.py b/tests/unittests/test_handler/test_handler_timezone.py
index c30fbdfe..27eedded 100644
--- a/tests/unittests/test_handler/test_handler_timezone.py
+++ b/tests/unittests/test_handler/test_handler_timezone.py
@@ -13,7 +13,7 @@ from cloudinit import util
 
 from cloudinit.sources import DataSourceNoCloud
 
-from .. import helpers as t_help
+from cloudinit.tests import helpers as t_help
 
 from configobj import ConfigObj
 import logging
diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py
index 1129e77d..7fa8fd21 100644
--- a/tests/unittests/test_handler/test_handler_write_files.py
+++ b/tests/unittests/test_handler/test_handler_write_files.py
@@ -4,7 +4,7 @@ from cloudinit.config.cc_write_files import write_files, decode_perms
 from cloudinit import log as logging
 from cloudinit import util
 
-from ..helpers import CiTestCase, FilesystemMockingTestCase
+from cloudinit.tests.helpers import CiTestCase, FilesystemMockingTestCase
 
 import base64
 import gzip
diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py
index c4396df5..b7adbe50 100644
--- a/tests/unittests/test_handler/test_handler_yum_add_repo.py
+++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py
@@ -3,7 +3,7 @@
 from cloudinit.config import cc_yum_add_repo
 from cloudinit import util
 
-from .. import helpers
+from cloudinit.tests import helpers
 
 try:
     from configparser import ConfigParser
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index 640f11d4..6137e3cf 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -6,7 +6,7 @@ from cloudinit.config.schema import (
     validate_cloudconfig_schema, main)
 from cloudinit.util import write_file
 
-from ..helpers import CiTestCase, mock, skipIf
+from cloudinit.tests.helpers import CiTestCase, mock, skipIf
 
 from copy import copy
 from six import StringIO
diff --git a/tests/unittests/test_helpers.py b/tests/unittests/test_helpers.py
index f1979e89..2e4582a0 100644
--- a/tests/unittests/test_helpers.py
+++ b/tests/unittests/test_helpers.py
@@ -4,7 +4,7 @@
 
 import os
 
-from . import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 
 from cloudinit import sources
 
diff --git a/tests/unittests/test_log.py b/tests/unittests/test_log.py
index 68fb4b8d..cd6296d6 100644
--- a/tests/unittests/test_log.py
+++ b/tests/unittests/test_log.py
@@ -2,9 +2,9 @@
 
 """Tests for cloudinit.log """
 
-from .helpers import CiTestCase
 from cloudinit.analyze.dump import CLOUD_INIT_ASCTIME_FMT
 from cloudinit import log as ci_logging
+from cloudinit.tests.helpers import CiTestCase
 import datetime
 import logging
 import six
diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index 0658b6b4..f51358da 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -1,6 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from . import helpers
+from cloudinit.tests import helpers
 
 from cloudinit.handlers import cloud_config
 from cloudinit.handlers import (CONTENT_START, CONTENT_END)
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index f251024b..c10ef905 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -11,10 +11,10 @@ from cloudinit.net import sysconfig
 from cloudinit.sources.helpers import openstack
 from cloudinit import util
 
-from .helpers import CiTestCase
-from .helpers import dir2dict
-from .helpers import mock
-from .helpers import populate_dir
+from cloudinit.tests.helpers import CiTestCase
+from cloudinit.tests.helpers import dir2dict
+from cloudinit.tests.helpers import mock
+from cloudinit.tests.helpers import populate_dir
 
 import base64
 import copy
diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py
index a4ae284f..abbb29b8 100644
--- a/tests/unittests/test_pathprefix2dict.py
+++ b/tests/unittests/test_pathprefix2dict.py
@@ -2,7 +2,7 @@
 
 from cloudinit import util
 
-from .helpers import TestCase, populate_dir
+from cloudinit.tests.helpers import TestCase, populate_dir
 
 import shutil
 import tempfile
diff --git a/tests/unittests/test_registry.py b/tests/unittests/test_registry.py
index acf0bf4f..2b625026 100644
--- a/tests/unittests/test_registry.py
+++ b/tests/unittests/test_registry.py
@@ -2,7 +2,7 @@
 
 from cloudinit.registry import DictRegistry
 
-from .helpers import (mock, TestCase)
+from cloudinit.tests.helpers import (mock, TestCase)
 
 
 class TestDictRegistry(TestCase):
diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py
index f3b8f992..571420ed 100644
--- a/tests/unittests/test_reporting.py
+++ b/tests/unittests/test_reporting.py
@@ -8,7 +8,7 @@ from cloudinit.reporting import handlers
 
 import mock
 
-from .helpers import TestCase
+from cloudinit.tests.helpers import TestCase
 
 
 def _fake_registry():
diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py
index ca14cd46..e9d5702a 100644
--- a/tests/unittests/test_rh_subscription.py
+++ b/tests/unittests/test_rh_subscription.py
@@ -7,7 +7,7 @@ import logging
 from cloudinit.config import cc_rh_subscription
 from cloudinit import util
 
-from .helpers import TestCase, mock
+from cloudinit.tests.helpers import TestCase, mock
 
 
 class GoodTests(TestCase):
diff --git a/tests/unittests/test_runs/test_merge_run.py b/tests/unittests/test_runs/test_merge_run.py
index 65895273..add93653 100644
--- a/tests/unittests/test_runs/test_merge_run.py
+++ b/tests/unittests/test_runs/test_merge_run.py
@@ -4,7 +4,7 @@ import os
 import shutil
 import tempfile
 
-from .. import helpers
+from cloudinit.tests import helpers
 
 from cloudinit.settings import PER_INSTANCE
 from cloudinit import stages
diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py
index 55f15b55..5cf666fe 100644
--- a/tests/unittests/test_runs/test_simple_run.py
+++ b/tests/unittests/test_runs/test_simple_run.py
@@ -4,7 +4,7 @@ import os
 import shutil
 import tempfile
 
-from .. import helpers
+from cloudinit.tests import helpers
 
 from cloudinit.settings import PER_INSTANCE
 from cloudinit import stages
diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py
index 991f45a6..2a8e6abe 100644
--- a/tests/unittests/test_sshutil.py
+++ b/tests/unittests/test_sshutil.py
@@ -2,8 +2,8 @@
 
 from mock import patch
 
-from . import helpers as test_helpers
 from cloudinit import ssh_util
+from cloudinit.tests import helpers as test_helpers
 
 
 VALID_CONTENT = {
@@ -57,6 +57,7 @@ TEST_OPTIONS = (
 
 
 class TestAuthKeyLineParser(test_helpers.TestCase):
+
     def test_simple_parse(self):
         # test key line with common 3 fields (keytype, base64, comment)
         parser = ssh_util.AuthKeyLineParser()
diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py
index 4e627826..b911d929 100644
--- a/tests/unittests/test_templating.py
+++ b/tests/unittests/test_templating.py
@@ -6,7 +6,7 @@
 
 from __future__ import print_function
 
-from . import helpers as test_helpers
+from cloudinit.tests import helpers as test_helpers
 import textwrap
 
 from cloudinit import templater
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 5f11c88f..3e4154ca 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -12,7 +12,7 @@ import six
 import yaml
 
 from cloudinit import importer, util
-from . import helpers
+from cloudinit.tests import helpers
 
 try:
     from unittest import mock
diff --git a/tests/unittests/test_version.py b/tests/unittests/test_version.py
index 1662ce09..d012f69d 100644
--- a/tests/unittests/test_version.py
+++ b/tests/unittests/test_version.py
@@ -1,6 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from .helpers import CiTestCase
+from cloudinit.tests.helpers import CiTestCase
 from cloudinit import version
 
 
diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py
index 03b36d31..d8651077 100644
--- a/tests/unittests/test_vmware_config_file.py
+++ b/tests/unittests/test_vmware_config_file.py
@@ -8,10 +8,10 @@
 import logging
 import sys
 
-from .helpers import CiTestCase
 from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProtoEnum
 from cloudinit.sources.helpers.vmware.imc.config import Config
 from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile
+from cloudinit.tests.helpers import CiTestCase
 
 logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)
 logger = logging.getLogger(__name__)
-- 
cgit v1.2.3


From e626966ee7d339b53d2c8b14a8f2ff8e3fe892ee Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Tue, 12 Sep 2017 10:27:07 -0600
Subject: cmdline: add collect-logs subcommand.

Add a new collect-logs sub command to the cloud-init CLI. This script
will collect all logs pertinent to a cloud-init run and store them in a
compressed tar-gzipped file. This tarfile can be attached to any
cloud-init bug filed in order to aid in bug triage and resolution.

A cloudinit.apport module is also added that allows apport interaction.
Here is an example bug filed via ubuntu-bug cloud-init: LP: #1716975.

Once the apport launcher is packaged in cloud-init, bugs can be filed
against cloud-init with the following command:
  ubuntu-bug cloud-init

LP: #1607345
---
 cloudinit/apport.py                    | 105 +++++++++++++++++++++++++++++
 cloudinit/cmd/devel/logs.py            | 101 +++++++++++++++++++++++++++
 cloudinit/cmd/devel/tests/__init__.py  |   0
 cloudinit/cmd/devel/tests/test_logs.py | 120 +++++++++++++++++++++++++++++++++
 cloudinit/cmd/main.py                  |  11 ++-
 packages/debian/rules.in               |   1 +
 tests/unittests/test_cli.py            |  22 ++++--
 7 files changed, 354 insertions(+), 6 deletions(-)
 create mode 100644 cloudinit/apport.py
 create mode 100644 cloudinit/cmd/devel/logs.py
 create mode 100644 cloudinit/cmd/devel/tests/__init__.py
 create mode 100644 cloudinit/cmd/devel/tests/test_logs.py

(limited to 'tests/unittests/test_cli.py')

diff --git a/cloudinit/apport.py b/cloudinit/apport.py
new file mode 100644
index 00000000..221f341c
--- /dev/null
+++ b/cloudinit/apport.py
@@ -0,0 +1,105 @@
+# Copyright (C) 2017 Canonical Ltd.
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+'''Cloud-init apport interface'''
+
+try:
+    from apport.hookutils import (
+        attach_file, attach_root_command_outputs, root_command_output)
+    has_apport = True
+except ImportError:
+    has_apport = False
+
+
+KNOWN_CLOUD_NAMES = [
+    'Amazon - Ec2', 'AliYun', 'AltCloud', 'Azure', 'Bigstep', 'CloudSigma',
+    'CloudStack', 'DigitalOcean', 'GCE - Google Compute Engine', 'MAAS',
+    'NoCloud', 'OpenNebula', 'OpenStack', 'OVF', 'Scaleway', 'SmartOS',
+    'VMware', 'Other']
+
+# Potentially clear text collected logs
+CLOUDINIT_LOG = '/var/log/cloud-init.log'
+CLOUDINIT_OUTPUT_LOG = '/var/log/cloud-init-output.log'
+USER_DATA_FILE = '/var/lib/cloud/instance/user-data.txt'  # Optional
+
+
+def attach_cloud_init_logs(report, ui=None):
+    '''Attach cloud-init logs and tarfile from 'cloud-init collect-logs'.'''
+    attach_root_command_outputs(report, {
+        'cloud-init-log-warnings':
+            'egrep -i "warn|error" /var/log/cloud-init.log',
+        'cloud-init-output.log.txt': 'cat /var/log/cloud-init-output.log'})
+    root_command_output(
+        ['cloud-init', 'collect-logs', '-t', '/tmp/cloud-init-logs.tgz'])
+    attach_file(report, '/tmp/cloud-init-logs.tgz', 'logs.tgz')
+
+
+def attach_hwinfo(report, ui=None):
+    '''Optionally attach hardware info from lshw.'''
+    prompt = (
+        'Your device details (lshw) may be useful to developers when'
+        ' addressing this bug, but gathering it requires admin privileges.'
+        ' Would you like to include this info?')
+    if ui and ui.yesno(prompt):
+        attach_root_command_outputs(report, {'lshw.txt': 'lshw'})
+
+
+def attach_cloud_info(report, ui=None):
+    '''Prompt for cloud details if available.'''
+    if ui:
+        prompt = 'Is this machine running in a cloud environment?'
+        response = ui.yesno(prompt)
+        if response is None:
+            raise StopIteration  # User cancelled
+        if response:
+            prompt = ('Please select the cloud vendor or environment in which'
+                      ' this instance is running')
+            response = ui.choice(prompt, KNOWN_CLOUD_NAMES)
+            if response:
+                report['CloudName'] = KNOWN_CLOUD_NAMES[response[0]]
+            else:
+                report['CloudName'] = 'None'
+
+
+def attach_user_data(report, ui=None):
+    '''Optionally provide user-data if desired.'''
+    if ui:
+        prompt = (
+            'Your user-data or cloud-config file can optionally be provided'
+            ' from {0} and could be useful to developers when addressing this'
+            ' bug. Do you wish to attach user-data to this bug?'.format(
+                USER_DATA_FILE))
+        response = ui.yesno(prompt)
+        if response is None:
+            raise StopIteration  # User cancelled
+        if response:
+            attach_file(report, USER_DATA_FILE, 'user_data.txt')
+
+
+def add_bug_tags(report):
+    '''Add any appropriate tags to the bug.'''
+    if 'JournalErrors' in report.keys():
+        errors = report['JournalErrors']
+        if 'Breaking ordering cycle' in errors:
+            report['Tags'] = 'systemd-ordering'
+
+
+def add_info(report, ui):
+    '''This is an entry point to run cloud-init's apport functionality.
+
+    Distros which want apport support will have a cloud-init package-hook at
+    /usr/share/apport/package-hooks/cloud-init.py which defines an add_info
+    function and returns the result of cloudinit.apport.add_info(report, ui).
+    '''
+    if not has_apport:
+        raise RuntimeError(
+            'No apport imports discovered. Apport functionality disabled')
+    attach_cloud_init_logs(report, ui)
+    attach_hwinfo(report, ui)
+    attach_cloud_info(report, ui)
+    attach_user_data(report, ui)
+    add_bug_tags(report)
+    return True
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py
new file mode 100644
index 00000000..35ca478f
--- /dev/null
+++ b/cloudinit/cmd/devel/logs.py
@@ -0,0 +1,101 @@
+# Copyright (C) 2017 Canonical Ltd.
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Define 'collect-logs' utility and handler to include in cloud-init cmd."""
+
+import argparse
+from cloudinit.util import (
+    ProcessExecutionError, chdir, copy, ensure_dir, subp, write_file)
+from cloudinit.temp_utils import tempdir
+from datetime import datetime
+import os
+import shutil
+
+
+CLOUDINIT_LOGS = ['/var/log/cloud-init.log', '/var/log/cloud-init-output.log']
+CLOUDINIT_RUN_DIR = '/run/cloud-init'
+USER_DATA_FILE = '/var/lib/cloud/instance/user-data.txt'  # Optional
+
+
+def get_parser(parser=None):
+    """Build or extend and arg parser for collect-logs utility.
+
+    @param parser: Optional existing ArgumentParser instance representing the
+        collect-logs subcommand which will be extended to support the args of
+        this utility.
+
+    @returns: ArgumentParser with proper argument configuration.
+    """
+    if not parser:
+        parser = argparse.ArgumentParser(
+            prog='collect-logs',
+            description='Collect and tar all cloud-init debug info')
+    parser.add_argument(
+        "--tarfile", '-t', default='cloud-init.tar.gz',
+        help=('The tarfile to create containing all collected logs.'
+              ' Default: cloud-init.tar.gz'))
+    parser.add_argument(
+        "--include-userdata", '-u', default=False, action='store_true',
+        dest='userdata', help=(
+            'Optionally include user-data from {0} which could contain'
+            ' sensitive information.'.format(USER_DATA_FILE)))
+    return parser
+
+
+def _write_command_output_to_file(cmd, filename):
+    """Helper which runs a command and writes output or error to filename."""
+    try:
+        out, _ = subp(cmd)
+    except ProcessExecutionError as e:
+        write_file(filename, str(e))
+    else:
+        write_file(filename, out)
+
+
+def collect_logs(tarfile, include_userdata):
+    """Collect all cloud-init logs and tar them up into the provided tarfile.
+
+    @param tarfile: The path of the tar-gzipped file to create.
+    @param include_userdata: Boolean, true means include user-data.
+    """
+    tarfile = os.path.abspath(tarfile)
+    date = datetime.utcnow().date().strftime('%Y-%m-%d')
+    log_dir = 'cloud-init-logs-{0}'.format(date)
+    with tempdir(dir='/tmp') as tmp_dir:
+        log_dir = os.path.join(tmp_dir, log_dir)
+        _write_command_output_to_file(
+            ['dpkg-query', '--show', "-f=${Version}\n", 'cloud-init'],
+            os.path.join(log_dir, 'version'))
+        _write_command_output_to_file(
+            ['dmesg'], os.path.join(log_dir, 'dmesg.txt'))
+        _write_command_output_to_file(
+            ['journalctl', '-o', 'short-precise'],
+            os.path.join(log_dir, 'journal.txt'))
+        for log in CLOUDINIT_LOGS:
+            copy(log, log_dir)
+        if include_userdata:
+            copy(USER_DATA_FILE, log_dir)
+        run_dir = os.path.join(log_dir, 'run')
+        ensure_dir(run_dir)
+        shutil.copytree(CLOUDINIT_RUN_DIR, os.path.join(run_dir, 'cloud-init'))
+        with chdir(tmp_dir):
+            subp(['tar', 'czvf', tarfile, log_dir.replace(tmp_dir + '/', '')])
+
+
+def handle_collect_logs_args(name, args):
+    """Handle calls to 'cloud-init collect-logs' as a subcommand."""
+    collect_logs(args.tarfile, args.userdata)
+
+
+def main():
+    """Tool to collect and tar all cloud-init related logs."""
+    parser = get_parser()
+    handle_collect_logs_args('collect-logs', parser.parse_args())
+    return 0
+
+
+if __name__ == '__main__':
+    main()
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/devel/tests/__init__.py b/cloudinit/cmd/devel/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/cloudinit/cmd/devel/tests/test_logs.py b/cloudinit/cmd/devel/tests/test_logs.py
new file mode 100644
index 00000000..dc4947cc
--- /dev/null
+++ b/cloudinit/cmd/devel/tests/test_logs.py
@@ -0,0 +1,120 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit.cmd.devel import logs
+from cloudinit.util import ensure_dir, load_file, subp, write_file
+from cloudinit.tests.helpers import FilesystemMockingTestCase, wrap_and_call
+from datetime import datetime
+import os
+
+
+class TestCollectLogs(FilesystemMockingTestCase):
+
+    def setUp(self):
+        super(TestCollectLogs, self).setUp()
+        self.new_root = self.tmp_dir()
+        self.run_dir = self.tmp_path('run', self.new_root)
+
+    def test_collect_logs_creates_tarfile(self):
+        """collect-logs creates a tarfile with all related cloud-init info."""
+        log1 = self.tmp_path('cloud-init.log', self.new_root)
+        write_file(log1, 'cloud-init-log')
+        log2 = self.tmp_path('cloud-init-output.log', self.new_root)
+        write_file(log2, 'cloud-init-output-log')
+        ensure_dir(self.run_dir)
+        write_file(self.tmp_path('results.json', self.run_dir), 'results')
+        output_tarfile = self.tmp_path('logs.tgz')
+
+        date = datetime.utcnow().date().strftime('%Y-%m-%d')
+        date_logdir = 'cloud-init-logs-{0}'.format(date)
+
+        expected_subp = {
+            ('dpkg-query', '--show', "-f=${Version}\n", 'cloud-init'):
+                '0.7fake\n',
+            ('dmesg',): 'dmesg-out\n',
+            ('journalctl', '-o', 'short-precise'): 'journal-out\n',
+            ('tar', 'czvf', output_tarfile, date_logdir): ''
+        }
+
+        def fake_subp(cmd):
+            cmd_tuple = tuple(cmd)
+            if cmd_tuple not in expected_subp:
+                raise AssertionError(
+                    'Unexpected command provided to subp: {0}'.format(cmd))
+            if cmd == ['tar', 'czvf', output_tarfile, date_logdir]:
+                subp(cmd)  # Pass through tar cmd so we can check output
+            return expected_subp[cmd_tuple], ''
+
+        wrap_and_call(
+            'cloudinit.cmd.devel.logs',
+            {'subp': {'side_effect': fake_subp},
+             'CLOUDINIT_LOGS': {'new': [log1, log2]},
+             'CLOUDINIT_RUN_DIR': {'new': self.run_dir}},
+            logs.collect_logs, output_tarfile, include_userdata=False)
+        # unpack the tarfile and check file contents
+        subp(['tar', 'zxvf', output_tarfile, '-C', self.new_root])
+        out_logdir = self.tmp_path(date_logdir, self.new_root)
+        self.assertEqual(
+            '0.7fake\n',
+            load_file(os.path.join(out_logdir, 'version')))
+        self.assertEqual(
+            'cloud-init-log',
+            load_file(os.path.join(out_logdir, 'cloud-init.log')))
+        self.assertEqual(
+            'cloud-init-output-log',
+            load_file(os.path.join(out_logdir, 'cloud-init-output.log')))
+        self.assertEqual(
+            'dmesg-out\n',
+            load_file(os.path.join(out_logdir, 'dmesg.txt')))
+        self.assertEqual(
+            'journal-out\n',
+            load_file(os.path.join(out_logdir, 'journal.txt')))
+        self.assertEqual(
+            'results',
+            load_file(
+                os.path.join(out_logdir, 'run', 'cloud-init', 'results.json')))
+
+    def test_collect_logs_includes_optional_userdata(self):
+        """collect-logs include userdata when --include-userdata is set."""
+        log1 = self.tmp_path('cloud-init.log', self.new_root)
+        write_file(log1, 'cloud-init-log')
+        log2 = self.tmp_path('cloud-init-output.log', self.new_root)
+        write_file(log2, 'cloud-init-output-log')
+        userdata = self.tmp_path('user-data.txt', self.new_root)
+        write_file(userdata, 'user-data')
+        ensure_dir(self.run_dir)
+        write_file(self.tmp_path('results.json', self.run_dir), 'results')
+        output_tarfile = self.tmp_path('logs.tgz')
+
+        date = datetime.utcnow().date().strftime('%Y-%m-%d')
+        date_logdir = 'cloud-init-logs-{0}'.format(date)
+
+        expected_subp = {
+            ('dpkg-query', '--show', "-f=${Version}\n", 'cloud-init'):
+                '0.7fake',
+            ('dmesg',): 'dmesg-out\n',
+            ('journalctl', '-o', 'short-precise'): 'journal-out\n',
+            ('tar', 'czvf', output_tarfile, date_logdir): ''
+        }
+
+        def fake_subp(cmd):
+            cmd_tuple = tuple(cmd)
+            if cmd_tuple not in expected_subp:
+                raise AssertionError(
+                    'Unexpected command provided to subp: {0}'.format(cmd))
+            if cmd == ['tar', 'czvf', output_tarfile, date_logdir]:
+                subp(cmd)  # Pass through tar cmd so we can check output
+            return expected_subp[cmd_tuple], ''
+
+        wrap_and_call(
+            'cloudinit.cmd.devel.logs',
+            {'subp': {'side_effect': fake_subp},
+             'CLOUDINIT_LOGS': {'new': [log1, log2]},
+             'CLOUDINIT_RUN_DIR': {'new': self.run_dir},
+             'USER_DATA_FILE': {'new': userdata}},
+            logs.collect_logs, output_tarfile, include_userdata=True)
+        # unpack the tarfile and check file contents
+        subp(['tar', 'zxvf', output_tarfile, '-C', self.new_root])
+        out_logdir = self.tmp_path(date_logdir, self.new_root)
+        self.assertEqual(
+            'user-data',
+            load_file(os.path.join(out_logdir, 'user-data.txt')))
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 68563e0c..6fb9d9e7 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -764,16 +764,25 @@ def main(sysv_args=None):
     parser_devel = subparsers.add_parser(
         'devel', help='Run development tools')
 
+    parser_collect_logs = subparsers.add_parser(
+        'collect-logs', help='Collect and tar all cloud-init debug info')
+
     if sysv_args:
         # Only load subparsers if subcommand is specified to avoid load cost
         if sysv_args[0] == 'analyze':
             from cloudinit.analyze.__main__ import get_parser as analyze_parser
             # Construct analyze subcommand parser
             analyze_parser(parser_analyze)
-        if sysv_args[0] == 'devel':
+        elif sysv_args[0] == 'devel':
             from cloudinit.cmd.devel.parser import get_parser as devel_parser
             # Construct devel subcommand parser
             devel_parser(parser_devel)
+        elif sysv_args[0] == 'collect-logs':
+            from cloudinit.cmd.devel.logs import (
+                get_parser as logs_parser, handle_collect_logs_args)
+            logs_parser(parser_collect_logs)
+            parser_collect_logs.set_defaults(
+                action=('collect-logs', handle_collect_logs_args))
 
     args = parser.parse_args(args=sysv_args)
 
diff --git a/packages/debian/rules.in b/packages/debian/rules.in
index b87a5e84..4aa907e3 100755
--- a/packages/debian/rules.in
+++ b/packages/debian/rules.in
@@ -10,6 +10,7 @@ PYVER ?= python${pyver}
 override_dh_install:
 	dh_install
 	install -d debian/cloud-init/etc/rsyslog.d
+	install -d debian/cloud-init/usr/share/apport/package-hooks
 	cp tools/21-cloudinit.conf debian/cloud-init/etc/rsyslog.d/21-cloudinit.conf
 	install -D ./tools/Z99-cloud-locale-test.sh debian/cloud-init/etc/profile.d/Z99-cloud-locale-test.sh
 	install -D ./tools/Z99-cloudinit-warnings.sh debian/cloud-init/etc/profile.d/Z99-cloudinit-warnings.sh
diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index 495bdc9f..258a9f08 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -72,18 +72,22 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
 
     def test_conditional_subcommands_from_entry_point_sys_argv(self):
         """Subcommands from entry-point are properly parsed from sys.argv."""
+        stdout = six.StringIO()
+        self.patchStdoutAndStderr(stdout=stdout)
+
         expected_errors = [
-            'usage: cloud-init analyze', 'usage: cloud-init devel']
-        conditional_subcommands = ['analyze', 'devel']
+            'usage: cloud-init analyze', 'usage: cloud-init collect-logs',
+            'usage: cloud-init devel']
+        conditional_subcommands = ['analyze', 'collect-logs', 'devel']
         # The cloud-init entrypoint calls main without passing sys_argv
         for subcommand in conditional_subcommands:
-            with mock.patch('sys.argv', ['cloud-init', subcommand]):
+            with mock.patch('sys.argv', ['cloud-init', subcommand, '-h']):
                 try:
                     cli.main()
                 except SystemExit as e:
-                    self.assertEqual(2, e.code)  # exit 2 on proper usage docs
+                    self.assertEqual(0, e.code)  # exit 2 on proper -h usage
         for error_message in expected_errors:
-            self.assertIn(error_message, self.stderr.getvalue())
+            self.assertIn(error_message, stdout.getvalue())
 
     def test_analyze_subcommand_parser(self):
         """The subcommand cloud-init analyze calls the correct subparser."""
@@ -94,6 +98,14 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
         for subcommand in expected_subcommands:
             self.assertIn(subcommand, error)
 
+    def test_collect_logs_subcommand_parser(self):
+        """The subcommand cloud-init collect-logs calls the subparser."""
+        # Provide -h param to collect-logs to avoid having to mock behavior.
+        stdout = six.StringIO()
+        self.patchStdoutAndStderr(stdout=stdout)
+        self._call_main(['cloud-init', 'collect-logs', '-h'])
+        self.assertIn('usage: cloud-init collect-log', stdout.getvalue())
+
     def test_devel_subcommand_parser(self):
         """The subcommand cloud-init devel calls the correct subparser."""
         self._call_main(['cloud-init', 'devel'])
-- 
cgit v1.2.3


From 27613443139578dd1b968d1f5f953bd2bc6245a4 Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Wed, 20 Sep 2017 23:41:22 -0600
Subject: docs: fix sphinx module schema documentation

Create a copy of each modules schema attribute when generating sphinx docs
to avoid altering the actual module dict in memory. This avoids illegible
rendering of module examples and distros where each character of a list
was represented on a separate line by itself.

Fixes ntp, resizefs, runcmd and bootcmd docs.
---
 cloudinit/config/schema.py  | 12 +++++++-----
 tests/unittests/test_cli.py | 15 +++++++++++++++
 2 files changed, 22 insertions(+), 5 deletions(-)

(limited to 'tests/unittests/test_cli.py')

diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py
index c17d973e..bb291ff8 100644
--- a/cloudinit/config/schema.py
+++ b/cloudinit/config/schema.py
@@ -8,6 +8,7 @@ from cloudinit.util import find_modules, read_file_or_url
 
 import argparse
 from collections import defaultdict
+from copy import deepcopy
 import logging
 import os
 import re
@@ -273,12 +274,13 @@ def get_schema_doc(schema):
     @param schema: Dict of jsonschema to render.
     @raise KeyError: If schema lacks an expected key.
     """
-    schema['property_doc'] = _get_property_doc(schema)
-    schema['examples'] = _get_schema_examples(schema)
-    schema['distros'] = ', '.join(schema['distros'])
+    schema_copy = deepcopy(schema)
+    schema_copy['property_doc'] = _get_property_doc(schema)
+    schema_copy['examples'] = _get_schema_examples(schema)
+    schema_copy['distros'] = ', '.join(schema['distros'])
     # Need an underbar of the same length as the name
-    schema['title_underbar'] = re.sub(r'.', '-', schema['name'])
-    return SCHEMA_DOC_TMPL.format(**schema)
+    schema_copy['title_underbar'] = re.sub(r'.', '-', schema['name'])
+    return SCHEMA_DOC_TMPL.format(**schema_copy)
 
 
 FULL_SCHEMA = None
diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index 258a9f08..fccbbd23 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -125,6 +125,21 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
             'Expected either --config-file argument or --doc\n',
             self.stderr.getvalue())
 
+    def test_wb_devel_schema_subcommand_doc_content(self):
+        """Validate that doc content is sane from known examples."""
+        stdout = six.StringIO()
+        self.patchStdoutAndStderr(stdout=stdout)
+        self._call_main(['cloud-init', 'devel', 'schema', '--doc'])
+        expected_doc_sections = [
+            '**Supported distros:** all',
+            '**Supported distros:** centos, debian, fedora',
+            '**Config schema**:\n    **resize_rootfs:** (true/false/noblock)',
+            '**Examples**::\n\n    runcmd:\n        - [ ls, -l, / ]\n'
+        ]
+        stdout = stdout.getvalue()
+        for expected in expected_doc_sections:
+            self.assertIn(expected, stdout)
+
     @mock.patch('cloudinit.cmd.main.main_single')
     def test_single_subcommand(self, m_main_single):
         """The subcommand 'single' calls main_single with valid args."""
-- 
cgit v1.2.3