schema cli: Add schema subcommand to cloud-init cli and cc_runcmd schema

This branch does a few things:
  - Add 'schema' subcommand to cloud-init CLI for validating
    cloud-config files against strict module jsonschema definitions
  - Add --annotate parameter to 'cloud-init schema' to annotate
    existing cloud-config file content with validation errors
  - Add jsonschema definition to cc_runcmd
  - Add unit test coverage for cc_runcmd
  - Update CLI capabilities documentation

This branch only imports development (and analyze) subparsers when the
specific subcommand is provided on the CLI to avoid adding costly unused
file imports during cloud-init system boot.

The schema command allows a person to quickly validate a cloud-config text
file against cloud-init's known module schemas to avoid costly roundtrips
deploying instances in their cloud of choice. As of this branch, only
cc_ntp and cc_runcmd cloud-config modules define schemas. Schema
validation will ignore all undefined config keys until all modules define
a strict schema.

To perform validation of runcmd and ntp sections of a cloud-config file:
$ cat > cloud.cfg <<EOF
runcmd: bogus
EOF
$ python -m cloudinit.cmd.main schema --config-file cloud.cfg

$ python -m cloudinit.cmd.main schema --config-file cloud.cfg \
  --annotate

Once jsonschema is defined for all ~55 cc modules, we will move this
schema subcommand up as a proper subcommand of the cloud-init CLI.

9 files changed, 541 insertions, 91 deletions

diff --git a/cloudinit/cmd/devel/__init__.py b/cloudinit/cmd/devel/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/cloudinit/cmd/devel/__init__.py
diff --git a/cloudinit/cmd/devel/parser.py b/cloudinit/cmd/devel/parser.py
new file mode 100644
index 00000000..acacc4ed
--- /dev/null
+++ b/cloudinit/cmd/devel/parser.py
@@ -0,0 +1,26 @@
+# Copyright (C) 2017 Canonical Ltd.
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Define 'devel' subcommand argument parsers to include in cloud-init cmd."""
+
+import argparse
+from cloudinit.config.schema import (
+    get_parser as schema_parser, handle_schema_args)
+
+
+def get_parser(parser=None):
+    if not parser:
+        parser = argparse.ArgumentParser(
+            prog='cloudinit-devel',
+            description='Run development cloud-init tools')
+    subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand')
+    subparsers.required = True
+
+    parser_schema = subparsers.add_parser(
+        'schema', help='Validate cloud-config files or document schema')
+    # Construct schema subcommand parser
+    schema_parser(parser_schema)
+    parser_schema.set_defaults(action=('schema', handle_schema_args))
+
+    return parser
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 9c0ac864..5b467979 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -705,7 +705,6 @@ def main(sysv_args=None):
     subparsers.required = True
 
     # Each action and its sub-options (if any)
-
     parser_init = subparsers.add_parser('init',
                                         help=('initializes cloud-init and'
                                               ' performs initial modules'))
@@ -762,12 +761,20 @@ def main(sysv_args=None):
 
     parser_analyze = subparsers.add_parser(
         'analyze', help='Devel tool: Analyze cloud-init logs and data')
-    if sysv_args and sysv_args[0] == 'analyze':
-        # Only load this parser if analyze is specified to avoid file load cost
-        # FIXME put this under 'devel' subcommand (coming in next branch)
-        from cloudinit.analyze.__main__ import get_parser as analyze_parser
-        # Construct analyze subcommand parser
-        analyze_parser(parser_analyze)
+
+    parser_devel = subparsers.add_parser(
+        'devel', help='Run development tools')
+
+    if sysv_args:
+        # Only load subparsers if subcommand is specified to avoid load cost
+        if sysv_args[0] == 'analyze':
+            from cloudinit.analyze.__main__ import get_parser as analyze_parser
+            # Construct analyze subcommand parser
+            analyze_parser(parser_analyze)
+        if sysv_args[0] == 'devel':
+            from cloudinit.cmd.devel.parser import get_parser as devel_parser
+            # Construct devel subcommand parser
+            devel_parser(parser_devel)
 
     args = parser.parse_args(args=sysv_args)
 
diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py
index dfa8cb3d..7c3ccd41 100644
--- a/cloudinit/config/cc_runcmd.py
+++ b/cloudinit/config/cc_runcmd.py
@@ -6,41 +6,66 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""
-Runcmd
-------
-**Summary:** run commands
+"""Runcmd: run arbitrary commands at rc.local with output to the console"""
 
-Run arbitrary commands at a rc.local like level with output to the console.
-Each item can be either a list or a string. If the item is a list, it will be
-properly executed as if passed to ``execve()`` (with the first arg as the
-command). If the item is a string, it will be written to a file and interpreted
-using ``sh``.
-
-.. note::
-    all commands must be proper yaml, so you have to quote any characters yaml
-    would eat (':' can be problematic)
-
-**Internal name:** ``cc_runcmd``
-
-**Module frequency:** per instance
+from cloudinit.config.schema import validate_cloudconfig_schema
+from cloudinit.settings import PER_INSTANCE
+from cloudinit import util
 
-**Supported distros:** all
+import os
+from textwrap import dedent
 
-**Config keys**::
 
-    runcmd:
-        - [ ls, -l, / ]
-        - [ sh, -xc, "echo $(date) ': hello world!'" ]
-        - [ sh, -c, echo "=========hello world'=========" ]
-        - ls -l /root
-        - [ wget, "http://example.org", -O, /tmp/index.html ]
-"""
+# The schema definition for each cloud-config module is a strict contract for
+# describing supported configuration parameters for each cloud-config section.
+# It allows cloud-config to validate and alert users to invalid or ignored
+# configuration options before actually attempting to deploy with said
+# configuration.
 
+distros = ['all']
 
-import os
+schema = {
+    'id': 'cc_runcmd',
+    'name': 'Runcmd',
+    'title': 'Run arbitrary commands',
+    'description': dedent("""\
+        Run arbitrary commands at a rc.local like level with output to the
+        console. Each item can be either a list or a string. If the item is a
+        list, it will be properly executed as if passed to ``execve()`` (with
+        the first arg as the command). If the item is a string, it will be
+        written to a file and interpreted
+        using ``sh``.
 
-from cloudinit import util
+        .. note::
+        all commands must be proper yaml, so you have to quote any characters
+        yaml would eat (':' can be problematic)"""),
+    'distros': distros,
+    'examples': [dedent("""\
+        runcmd:
+            - [ ls, -l, / ]
+            - [ sh, -xc, "echo $(date) ': hello world!'" ]
+            - [ sh, -c, echo "=========hello world'=========" ]
+            - ls -l /root
+            - [ wget, "http://example.org", -O, /tmp/index.html ]
+    """)],
+    'frequency': PER_INSTANCE,
+    'type': 'object',
+    'properties': {
+        'runcmd': {
+            'type': 'array',
+            'items': {
+                'oneOf': [
+                    {'type': 'array', 'items': {'type': 'string'}},
+                    {'type': 'string'}]
+            },
+            'additionalItems': False,  # Reject items of non-string non-list
+            'additionalProperties': False,
+            'minItems': 1,
+            'required': [],
+            'uniqueItems': True
+        }
+    }
+}
 
 
 def handle(name, cfg, cloud, log, _args):
@@ -49,6 +74,7 @@ def handle(name, cfg, cloud, log, _args):
                    " no 'runcmd' key in configuration"), name)
         return
 
+    validate_cloudconfig_schema(cfg, schema)
     out_fn = os.path.join(cloud.get_ipath('scripts'), "runcmd")
     cmd = cfg["runcmd"]
     try:
diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py
index 6400f005..73dd5c2e 100644
--- a/cloudinit/config/schema.py
+++ b/cloudinit/config/schema.py
@@ -3,11 +3,14 @@
 
 from __future__ import print_function
 
-from cloudinit.util import read_file_or_url
+from cloudinit import importer
+from cloudinit.util import find_modules, read_file_or_url
 
 import argparse
+from collections import defaultdict
 import logging
 import os
+import re
 import sys
 import yaml
 
@@ -15,7 +18,7 @@ SCHEMA_UNDEFINED = b'UNDEFINED'
 CLOUD_CONFIG_HEADER = b'#cloud-config'
 SCHEMA_DOC_TMPL = """
 {name}
----
+{title_underbar}
 **Summary:** {title}
 
 {description}
@@ -83,11 +86,49 @@ def validate_cloudconfig_schema(config, schema, strict=False):
             logging.warning('Invalid config:\n%s', '\n'.join(messages))
 
 
-def validate_cloudconfig_file(config_path, schema):
+def annotated_cloudconfig_file(cloudconfig, original_content, schema_errors):
+    """Return contents of the cloud-config file annotated with schema errors.
+
+    @param cloudconfig: YAML-loaded object from the original_content.
+    @param original_content: The contents of a cloud-config file
+    @param schema_errors: List of tuples from a JSONSchemaValidationError. The
+        tuples consist of (schemapath, error_message).
+    """
+    if not schema_errors:
+        return original_content
+    schemapaths = _schemapath_for_cloudconfig(cloudconfig, original_content)
+    errors_by_line = defaultdict(list)
+    error_count = 1
+    error_footer = []
+    annotated_content = []
+    for path, msg in schema_errors:
+        errors_by_line[schemapaths[path]].append(msg)
+        error_footer.append('# E{0}: {1}'.format(error_count, msg))
+        error_count += 1
+    lines = original_content.decode().split('\n')
+    error_count = 1
+    for line_number, line in enumerate(lines):
+        errors = errors_by_line[line_number + 1]
+        if errors:
+            error_label = ','.join(
+                ['E{0}'.format(count + error_count)
+                 for count in range(0, len(errors))])
+            error_count += len(errors)
+            annotated_content.append(line + '\t\t# ' + error_label)
+        else:
+            annotated_content.append(line)
+    annotated_content.append(
+        '# Errors: -------------\n{0}\n\n'.format('\n'.join(error_footer)))
+    return '\n'.join(annotated_content)
+
+
+def validate_cloudconfig_file(config_path, schema, annotate=False):
     """Validate cloudconfig file adheres to a specific jsonschema.
 
     @param config_path: Path to the yaml cloud-config file to parse.
     @param schema: Dict describing a valid jsonschema to validate against.
+    @param annotate: Boolean set True to print original config file with error
+        annotations on the offending lines.
 
     @raises SchemaValidationError containing any of schema_errors encountered.
     @raises RuntimeError when config_path does not exist.
@@ -108,8 +149,64 @@ def validate_cloudconfig_file(config_path, schema):
             ('format', 'File {0} is not valid yaml. {1}'.format(
                 config_path, str(e))),)
         raise SchemaValidationError(errors)
-    validate_cloudconfig_schema(
-        cloudconfig, schema, strict=True)
+
+    try:
+        validate_cloudconfig_schema(
+            cloudconfig, schema, strict=True)
+    except SchemaValidationError as e:
+        if annotate:
+            print(annotated_cloudconfig_file(
+                cloudconfig, content, e.schema_errors))
+        raise
+
+
+def _schemapath_for_cloudconfig(config, original_content):
+    """Return a dictionary mapping schemapath to original_content line number.
+
+    @param config: The yaml.loaded config dictionary of a cloud-config file.
+    @param original_content: The simple file content of the cloud-config file
+    """
+    # FIXME Doesn't handle multi-line lists or multi-line strings
+    content_lines = original_content.decode().split('\n')
+    schema_line_numbers = {}
+    list_index = 0
+    RE_YAML_INDENT = r'^(\s*)'
+    scopes = []
+    for line_number, line in enumerate(content_lines):
+        indent_depth = len(re.match(RE_YAML_INDENT, line).groups()[0])
+        line = line.strip()
+        if not line or line.startswith('#'):
+            continue
+        if scopes:
+            previous_depth, path_prefix = scopes[-1]
+        else:
+            previous_depth = -1
+            path_prefix = ''
+        if line.startswith('- '):
+            key = str(list_index)
+            value = line[1:]
+            list_index += 1
+        else:
+            list_index = 0
+            key, value = line.split(':', 1)
+        while indent_depth <= previous_depth:
+            if scopes:
+                previous_depth, path_prefix = scopes.pop()
+            else:
+                previous_depth = -1
+                path_prefix = ''
+        if path_prefix:
+            key = path_prefix + '.' + key
+        scopes.append((indent_depth, key))
+        if value:
+            value = value.strip()
+            if value.startswith('['):
+                scopes.append((indent_depth + 2, key + '.0'))
+                for inner_list_index in range(0, len(yaml.safe_load(value))):
+                    list_key = key + '.' + str(inner_list_index)
+                    schema_line_numbers[list_key] = line_number + 1
+        schema_line_numbers[key] = line_number + 1
+    return schema_line_numbers
 
 
 def _get_property_type(property_dict):
@@ -117,9 +214,15 @@ def _get_property_type(property_dict):
     property_type = property_dict.get('type', SCHEMA_UNDEFINED)
     if isinstance(property_type, list):
         property_type = '/'.join(property_type)
-    item_type = property_dict.get('items', {}).get('type')
-    if item_type:
-        property_type = '{0} of {1}'.format(property_type, item_type)
+    items = property_dict.get('items', {})
+    sub_property_type = items.get('type', '')
+    # Collect each item type
+    for sub_item in items.get('oneOf', {}):
+        if sub_property_type:
+            sub_property_type += '/'
+        sub_property_type += '(' + _get_property_type(sub_item) + ')'
+    if sub_property_type:
+        return '{0} of {1}'.format(property_type, sub_property_type)
     return property_type
 
 
@@ -148,9 +251,12 @@ def _get_schema_examples(schema, prefix=''):
         return ''
     rst_content = '\n**Examples**::\n\n'
     for example in examples:
-        example_yaml = yaml.dump(example, default_flow_style=False)
+        if isinstance(example, str):
+            example_content = example
+        else:
+            example_content = yaml.dump(example, default_flow_style=False)
         # Python2.6 is missing textwrapper.indent
-        lines = example_yaml.split('\n')
+        lines = example_content.split('\n')
         indented_lines = ['    {0}'.format(line) for line in lines]
         rst_content += '\n'.join(indented_lines)
     return rst_content
@@ -165,58 +271,83 @@ def get_schema_doc(schema):
     schema['property_doc'] = _get_property_doc(schema)
     schema['examples'] = _get_schema_examples(schema)
     schema['distros'] = ', '.join(schema['distros'])
+    # Need an underbar of the same length as the name
+    schema['title_underbar'] = re.sub(r'.', '-', schema['name'])
     return SCHEMA_DOC_TMPL.format(**schema)
 
 
-def get_schema(section_key=None):
-    """Return a dict of jsonschema defined in any cc_* module.
+FULL_SCHEMA = None
 
-    @param: section_key: Optionally limit schema to a specific top-level key.
-    """
-    # TODO use util.find_modules in subsequent branch
-    from cloudinit.config.cc_ntp import schema
-    return schema
+
+def get_schema():
+    """Return jsonschema coalesced from all cc_* cloud-config module."""
+    global FULL_SCHEMA
+    if FULL_SCHEMA:
+        return FULL_SCHEMA
+    full_schema = {
+        '$schema': 'http://json-schema.org/draft-04/schema#',
+        'id': 'cloud-config-schema', 'allOf': []}
+
+    configs_dir = os.path.dirname(os.path.abspath(__file__))
+    potential_handlers = find_modules(configs_dir)
+    for (fname, mod_name) in potential_handlers.items():
+        mod_locs, looked_locs = importer.find_module(
+            mod_name, ['cloudinit.config'], ['schema'])
+        if mod_locs:
+            mod = importer.import_module(mod_locs[0])
+            full_schema['allOf'].append(mod.schema)
+    FULL_SCHEMA = full_schema
+    return full_schema
 
 
 def error(message):
     print(message, file=sys.stderr)
-    return 1
+    sys.exit(1)
 
 
-def get_parser():
+def get_parser(parser=None):
     """Return a parser for supported cmdline arguments."""
-    parser = argparse.ArgumentParser()
+    if not parser:
+        parser = argparse.ArgumentParser(
+            prog='cloudconfig-schema',
+            description='Validate cloud-config files or document schema')
     parser.add_argument('-c', '--config-file',
                         help='Path of the cloud-config yaml file to validate')
     parser.add_argument('-d', '--doc', action="store_true", default=False,
                         help='Print schema documentation')
-    parser.add_argument('-k', '--key',
-                        help='Limit validation or docs to a section key')
+    parser.add_argument('--annotate', action="store_true", default=False,
+                        help='Annotate existing cloud-config file with errors')
     return parser
 
 
-def main():
-    """Tool to validate schema of a cloud-config file or print schema docs."""
-    parser = get_parser()
-    args = parser.parse_args()
+def handle_schema_args(name, args):
+    """Handle provided schema args and perform the appropriate actions."""
     exclusive_args = [args.config_file, args.doc]
     if not any(exclusive_args) or all(exclusive_args):
-        return error('Expected either --config-file argument or --doc')
-
-    schema = get_schema()
+        error('Expected either --config-file argument or --doc')
+    full_schema = get_schema()
     if args.config_file:
         try:
-            validate_cloudconfig_file(args.config_file, schema)
+            validate_cloudconfig_file(
+                args.config_file, full_schema, args.annotate)
         except (SchemaValidationError, RuntimeError) as e:
-            return error(str(e))
-        print("Valid cloud-config file {0}".format(args.config_file))
+            if not args.annotate:
+                error(str(e))
+        else:
+            print("Valid cloud-config file {0}".format(args.config_file))
     if args.doc:
-        print(get_schema_doc(schema))
+        for subschema in full_schema['allOf']:
+            print(get_schema_doc(subschema))
+
+
+def main():
+    """Tool to validate schema of a cloud-config file or print schema docs."""
+    parser = get_parser()
+    handle_schema_args('cloudconfig-schema', parser.parse_args())
     return 0
 
 
 if __name__ == '__main__':
     sys.exit(main())
 
-
 # vi: ts=4 expandtab
diff --git a/doc/rtd/topics/capabilities.rst b/doc/rtd/topics/capabilities.rst
index b8034b07..31eaba53 100644
--- a/doc/rtd/topics/capabilities.rst
+++ b/doc/rtd/topics/capabilities.rst
@@ -51,15 +51,6 @@ described in this document.
   usage: cloud-init [-h] [--version] [--file FILES] [--debug] [--force]
                     {init,modules,query,single,dhclient-hook,features} ...
 
-  positional arguments:
-    {init,modules,query,single,dhclient-hook,features}
-      init                initializes cloud-init and performs initial modules
-      modules             activates modules using a given configuration key
-      query               query information stored in cloud-init
-      single              run a single module
-      dhclient-hook       run the dhclient hookto record network info
-      features            list defined features
-
   optional arguments:
     -h, --help            show this help message and exit
     --version, -v         show program's version number and exit
@@ -69,6 +60,15 @@ described in this document.
     --force               force running even if no datasource is found (use at
                           your own risk)
 
+  Subcommands:
+    {init,modules,single,dhclient-hook,features,analyze,devel}
+      init                initializes cloud-init and performs initial modules
+      modules             activates modules using a given configuration key
+      single              run a single module
+      dhclient-hook       run the dhclient hookto record network info
+      features            list defined features
+      analyze             Devel tool: Analyze cloud-init logs and data
+      devel               Run development tools
 
   % cloud-init features
   NETWORK_CONFIG_V1
diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index 7780f164..24498802 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -46,7 +46,7 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
         self._call_main()
         error = self.stderr.getvalue()
         expected_subcommands = ['analyze', 'init', 'modules', 'single',
-                                'dhclient-hook', 'features']
+                                'dhclient-hook', 'features', 'devel']
         for subcommand in expected_subcommands:
             self.assertIn(subcommand, error)
 
@@ -79,6 +79,25 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
         for subcommand in expected_subcommands:
             self.assertIn(subcommand, error)
 
+    def test_devel_subcommand_parser(self):
+        """The subcommand cloud-init devel calls the correct subparser."""
+        self._call_main(['cloud-init', 'devel'])
+        # These subcommands only valid for cloud-init schema script
+        expected_subcommands = ['schema']
+        error = self.stderr.getvalue()
+        for subcommand in expected_subcommands:
+            self.assertIn(subcommand, error)
+
+    @mock.patch('cloudinit.config.schema.handle_schema_args')
+    def test_wb_devel_schema_subcommand_parser(self, m_schema):
+        """The subcommand cloud-init schema calls the correct subparser."""
+        exit_code = self._call_main(['cloud-init', 'devel', 'schema'])
+        self.assertEqual(1, exit_code)
+        # Known whitebox output from schema subcommand
+        self.assertEqual(
+            'Expected either --config-file argument or --doc\n',
+            self.stderr.getvalue())
+
     @mock.patch('cloudinit.cmd.main.main_single')
     def test_single_subcommand(self, m_main_single):
         """The subcommand 'single' calls main_single with valid args."""
diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py
new file mode 100644
index 00000000..7880ee72
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_runcmd.py
@@ -0,0 +1,108 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit.config import cc_runcmd
+from cloudinit.sources import DataSourceNone
+from cloudinit import (distros, helpers, cloud, util)
+from ..helpers import FilesystemMockingTestCase, skipIf
+
+import logging
+import os
+import stat
+
+try:
+    import jsonschema
+    assert jsonschema  # avoid pyflakes error F401: import unused
+    _missing_jsonschema_dep = False
+except ImportError:
+    _missing_jsonschema_dep = True
+
+LOG = logging.getLogger(__name__)
+
+
+class TestRuncmd(FilesystemMockingTestCase):
+
+    with_logs = True
+
+    def setUp(self):
+        super(TestRuncmd, self).setUp()
+        self.subp = util.subp
+        self.new_root = self.tmp_dir()
+
+    def _get_cloud(self, distro):
+        self.patchUtils(self.new_root)
+        paths = helpers.Paths({'scripts': self.new_root})
+        cls = distros.fetch(distro)
+        mydist = cls(distro, {}, paths)
+        myds = DataSourceNone.DataSourceNone({}, mydist, paths)
+        paths.datasource = myds
+        return cloud.Cloud(myds, paths, {}, mydist, None)
+
+    def test_handler_skip_if_no_runcmd(self):
+        """When the provided config doesn't contain runcmd, skip it."""
+        cfg = {}
+        mycloud = self._get_cloud('ubuntu')
+        cc_runcmd.handle('notimportant', cfg, mycloud, LOG, None)
+        self.assertIn(
+            "Skipping module named notimportant, no 'runcmd' key",
+            self.logs.getvalue())
+
+    def test_handler_invalid_command_set(self):
+        """Commands which can't be converted to shell will raise errors."""
+        invalid_config = {'runcmd': 1}
+        cc = self._get_cloud('ubuntu')
+        cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, [])
+        self.assertIn(
+            'Failed to shellify 1 into file'
+            ' /var/lib/cloud/instances/iid-datasource-none/scripts/runcmd',
+            self.logs.getvalue())
+
+    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    def test_handler_schema_validation_warns_non_array_type(self):
+        """Schema validation warns of non-array type for runcmd key.
+
+        Schema validation is not strict, so runcmd attempts to shellify the
+        invalid content.
+        """
+        invalid_config = {'runcmd': 1}
+        cc = self._get_cloud('ubuntu')
+        cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, [])
+        self.assertIn(
+            'Invalid config:\nruncmd: 1 is not of type \'array\'',
+            self.logs.getvalue())
+        self.assertIn('Failed to shellify', self.logs.getvalue())
+
+    @skipIf(_missing_jsonschema_dep, 'No python-jsonschema dependency')
+    def test_handler_schema_validation_warns_non_array_item_type(self):
+        """Schema validation warns of non-array or string runcmd items.
+
+        Schema validation is not strict, so runcmd attempts to shellify the
+        invalid content.
+        """
+        invalid_config = {
+            'runcmd': ['ls /', 20, ['wget', 'http://stuff/blah'], {'a': 'n'}]}
+        cc = self._get_cloud('ubuntu')
+        cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, [])
+        expected_warnings = [
+            'runcmd.1: 20 is not valid under any of the given schemas',
+            'runcmd.3: {\'a\': \'n\'} is not valid under any of the given'
+            ' schema'
+        ]
+        logs = self.logs.getvalue()
+        for warning in expected_warnings:
+            self.assertIn(warning, logs)
+        self.assertIn('Failed to shellify', logs)
+
+    def test_handler_write_valid_runcmd_schema_to_file(self):
+        """Valid runcmd schema is written to a runcmd shell script."""
+        valid_config = {'runcmd': [['ls', '/']]}
+        cc = self._get_cloud('ubuntu')
+        cc_runcmd.handle('cc_runcmd', valid_config, cc, LOG, [])
+        runcmd_file = os.path.join(
+            self.new_root,
+            'var/lib/cloud/instances/iid-datasource-none/scripts/runcmd')
+        self.assertEqual("#!/bin/sh\n'ls' '/'\n", util.load_file(runcmd_file))
+        file_stat = os.stat(runcmd_file)
+        self.assertEqual(0o700, stat.S_IMODE(file_stat.st_mode))
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index eda4802a..640f11d4 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -1,9 +1,9 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit.config.schema import (
-    CLOUD_CONFIG_HEADER, SchemaValidationError, get_schema_doc,
-    validate_cloudconfig_file, validate_cloudconfig_schema,
-    main)
+    CLOUD_CONFIG_HEADER, SchemaValidationError, annotated_cloudconfig_file,
+    get_schema_doc, get_schema, validate_cloudconfig_file,
+    validate_cloudconfig_schema, main)
 from cloudinit.util import write_file
 
 from ..helpers import CiTestCase, mock, skipIf
@@ -11,6 +11,7 @@ from ..helpers import CiTestCase, mock, skipIf
 from copy import copy
 from six import StringIO
 from textwrap import dedent
+from yaml import safe_load
 
 try:
     import jsonschema
@@ -20,6 +21,29 @@ except ImportError:
     _missing_jsonschema_dep = True
 
 
+class GetSchemaTest(CiTestCase):
+
+    def test_get_schema_coalesces_known_schema(self):
+        """Every cloudconfig module with schema is listed in allOf keyword."""
+        schema = get_schema()
+        self.assertItemsEqual(
+            ['cc_ntp', 'cc_runcmd'],
+            [subschema['id'] for subschema in schema['allOf']])
+        self.assertEqual('cloud-config-schema', schema['id'])
+        self.assertEqual(
+            'http://json-schema.org/draft-04/schema#',
+            schema['$schema'])
+        # FULL_SCHEMA is updated by the get_schema call
+        from cloudinit.config.schema import FULL_SCHEMA
+        self.assertItemsEqual(['id', '$schema', 'allOf'], FULL_SCHEMA.keys())
+
+    def test_get_schema_returns_global_when_set(self):
+        """When FULL_SCHEMA global is already set, get_schema returns it."""
+        m_schema_path = 'cloudinit.config.schema.FULL_SCHEMA'
+        with mock.patch(m_schema_path, {'here': 'iam'}):
+            self.assertEqual({'here': 'iam'}, get_schema())
+
+
 class SchemaValidationErrorTest(CiTestCase):
     """Test validate_cloudconfig_schema"""
 
@@ -151,11 +175,11 @@ class GetSchemaDocTest(CiTestCase):
         full_schema.update(
             {'properties': {
                 'prop1': {'type': 'array', 'description': 'prop-description',
-                          'items': {'type': 'int'}}}})
+                          'items': {'type': 'integer'}}}})
         self.assertEqual(
             dedent("""
                 name
-                ---
+                ----
                 **Summary:** title
 
                 description
@@ -167,27 +191,71 @@ class GetSchemaDocTest(CiTestCase):
                 **Supported distros:** debian, rhel
 
                 **Config schema**:
-                    **prop1:** (array of int) prop-description\n\n"""),
+                    **prop1:** (array of integer) prop-description\n\n"""),
+            get_schema_doc(full_schema))
+
+    def test_get_schema_doc_handles_multiple_types(self):
+        """get_schema_doc delimits multiple property types with a '/'."""
+        full_schema = copy(self.required_schema)
+        full_schema.update(
+            {'properties': {
+                'prop1': {'type': ['string', 'integer'],
+                          'description': 'prop-description'}}})
+        self.assertIn(
+            '**prop1:** (string/integer) prop-description',
+            get_schema_doc(full_schema))
+
+    def test_get_schema_doc_handles_nested_oneof_property_types(self):
+        """get_schema_doc describes array items oneOf declarations in type."""
+        full_schema = copy(self.required_schema)
+        full_schema.update(
+            {'properties': {
+                'prop1': {'type': 'array',
+                          'items': {
+                              'oneOf': [{'type': 'string'},
+                                        {'type': 'integer'}]},
+                          'description': 'prop-description'}}})
+        self.assertIn(
+            '**prop1:** (array of (string)/(integer)) prop-description',
             get_schema_doc(full_schema))
 
     def test_get_schema_doc_returns_restructured_text_with_examples(self):
         """get_schema_doc returns indented examples when present in schema."""
         full_schema = copy(self.required_schema)
         full_schema.update(
-            {'examples': {'ex1': [1, 2, 3]},
+            {'examples': [{'ex1': [1, 2, 3]}],
              'properties': {
                 'prop1': {'type': 'array', 'description': 'prop-description',
-                          'items': {'type': 'int'}}}})
+                          'items': {'type': 'integer'}}}})
         self.assertIn(
             dedent("""
                 **Config schema**:
-                    **prop1:** (array of int) prop-description
+                    **prop1:** (array of integer) prop-description
 
                 **Examples**::
 
                     ex1"""),
             get_schema_doc(full_schema))
 
+    def test_get_schema_doc_handles_unstructured_examples(self):
+        """get_schema_doc properly indented examples which as just strings."""
+        full_schema = copy(self.required_schema)
+        full_schema.update(
+            {'examples': ['My example:\n    [don\'t, expand, "this"]'],
+             'properties': {
+                'prop1': {'type': 'array', 'description': 'prop-description',
+                          'items': {'type': 'integer'}}}})
+        self.assertIn(
+            dedent("""
+                **Config schema**:
+                    **prop1:** (array of integer) prop-description
+
+                **Examples**::
+
+                    My example:
+                        [don't, expand, "this"]"""),
+            get_schema_doc(full_schema))
+
     def test_get_schema_doc_raises_key_errors(self):
         """get_schema_doc raises KeyErrors on missing keys."""
         for key in self.required_schema:
@@ -198,13 +266,78 @@ class GetSchemaDocTest(CiTestCase):
             self.assertIn(key, str(context_mgr.exception))
 
 
+class AnnotatedCloudconfigFileTest(CiTestCase):
+    maxDiff = None
+
+    def test_annotated_cloudconfig_file_no_schema_errors(self):
+        """With no schema_errors, print the original content."""
+        content = b'ntp:\n  pools: [ntp1.pools.com]\n'
+        self.assertEqual(
+            content,
+            annotated_cloudconfig_file({}, content, schema_errors=[]))
+
+    def test_annotated_cloudconfig_file_schema_annotates_and_adds_footer(self):
+        """With schema_errors, error lines are annotated and a footer added."""
+        content = dedent("""\
+            #cloud-config
+            # comment
+            ntp:
+              pools: [-99, 75]
+            """).encode()
+        expected = dedent("""\
+            #cloud-config
+            # comment
+            ntp:		# E1
+              pools: [-99, 75]		# E2,E3
+
+            # Errors: -------------
+            # E1: Some type error
+            # E2: -99 is not a string
+            # E3: 75 is not a string
+
+            """)
+        parsed_config = safe_load(content[13:])
+        schema_errors = [
+            ('ntp', 'Some type error'), ('ntp.pools.0', '-99 is not a string'),
+            ('ntp.pools.1', '75 is not a string')]
+        self.assertEqual(
+            expected,
+            annotated_cloudconfig_file(parsed_config, content, schema_errors))
+
+    def test_annotated_cloudconfig_file_annotates_separate_line_items(self):
+        """Errors are annotated for lists with items on separate lines."""
+        content = dedent("""\
+            #cloud-config
+            # comment
+            ntp:
+              pools:
+                - -99
+                - 75
+            """).encode()
+        expected = dedent("""\
+            ntp:
+              pools:
+                - -99		# E1
+                - 75		# E2
+            """)
+        parsed_config = safe_load(content[13:])
+        schema_errors = [
+            ('ntp.pools.0', '-99 is not a string'),
+            ('ntp.pools.1', '75 is not a string')]
+        self.assertIn(
+            expected,
+            annotated_cloudconfig_file(parsed_config, content, schema_errors))
+
+
 class MainTest(CiTestCase):
 
     def test_main_missing_args(self):
         """Main exits non-zero and reports an error on missing parameters."""
         with mock.patch('sys.argv', ['mycmd']):
             with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
-                self.assertEqual(1, main(), 'Expected non-zero exit code')
+                with self.assertRaises(SystemExit) as context_manager:
+                    main()
+        self.assertEqual('1', str(context_manager.exception))
         self.assertEqual(
             'Expected either --config-file argument or --doc\n',
             m_stderr.getvalue())
@@ -216,13 +349,13 @@ class MainTest(CiTestCase):
             with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
                 self.assertEqual(0, main(), 'Expected 0 exit code')
         self.assertIn('\nNTP\n---\n', m_stdout.getvalue())
+        self.assertIn('\nRuncmd\n------\n', m_stdout.getvalue())
 
     def test_main_validates_config_file(self):
         """When --config-file parameter is provided, main validates schema."""
         myyaml = self.tmp_path('my.yaml')
         myargs = ['mycmd', '--config-file', myyaml]
-        with open(myyaml, 'wb') as stream:
-            stream.write(b'#cloud-config\nntp:')   # shortest ntp schema
+        write_file(myyaml, b'#cloud-config\nntp:')  # shortest ntp schema
         with mock.patch('sys.argv', myargs):
             with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
                 self.assertEqual(0, main(), 'Expected 0 exit code')