schema cli: Add schema subcommand to cloud-init cli and cc_runcmd schema

This branch does a few things:
  - Add 'schema' subcommand to cloud-init CLI for validating
    cloud-config files against strict module jsonschema definitions
  - Add --annotate parameter to 'cloud-init schema' to annotate
    existing cloud-config file content with validation errors
  - Add jsonschema definition to cc_runcmd
  - Add unit test coverage for cc_runcmd
  - Update CLI capabilities documentation

This branch only imports development (and analyze) subparsers when the
specific subcommand is provided on the CLI to avoid adding costly unused
file imports during cloud-init system boot.

The schema command allows a person to quickly validate a cloud-config text
file against cloud-init's known module schemas to avoid costly roundtrips
deploying instances in their cloud of choice. As of this branch, only
cc_ntp and cc_runcmd cloud-config modules define schemas. Schema
validation will ignore all undefined config keys until all modules define
a strict schema.

To perform validation of runcmd and ntp sections of a cloud-config file:
$ cat > cloud.cfg <<EOF
runcmd: bogus
EOF
$ python -m cloudinit.cmd.main schema --config-file cloud.cfg

$ python -m cloudinit.cmd.main schema --config-file cloud.cfg \
  --annotate

Once jsonschema is defined for all ~55 cc modules, we will move this
schema subcommand up as a proper subcommand of the cloud-init CLI.

5 files changed, 259 insertions, 69 deletions

diff --git a/cloudinit/cmd/devel/__init__.py b/cloudinit/cmd/devel/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/cloudinit/cmd/devel/__init__.py
diff --git a/cloudinit/cmd/devel/parser.py b/cloudinit/cmd/devel/parser.py
new file mode 100644
index 00000000..acacc4ed
--- /dev/null
+++ b/cloudinit/cmd/devel/parser.py
@@ -0,0 +1,26 @@
+# Copyright (C) 2017 Canonical Ltd.
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Define 'devel' subcommand argument parsers to include in cloud-init cmd."""
+
+import argparse
+from cloudinit.config.schema import (
+    get_parser as schema_parser, handle_schema_args)
+
+
+def get_parser(parser=None):
+    if not parser:
+        parser = argparse.ArgumentParser(
+            prog='cloudinit-devel',
+            description='Run development cloud-init tools')
+    subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand')
+    subparsers.required = True
+
+    parser_schema = subparsers.add_parser(
+        'schema', help='Validate cloud-config files or document schema')
+    # Construct schema subcommand parser
+    schema_parser(parser_schema)
+    parser_schema.set_defaults(action=('schema', handle_schema_args))
+
+    return parser
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 9c0ac864..5b467979 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -705,7 +705,6 @@ def main(sysv_args=None):
     subparsers.required = True
 
     # Each action and its sub-options (if any)
-
     parser_init = subparsers.add_parser('init',
                                         help=('initializes cloud-init and'
                                               ' performs initial modules'))
@@ -762,12 +761,20 @@ def main(sysv_args=None):
 
     parser_analyze = subparsers.add_parser(
         'analyze', help='Devel tool: Analyze cloud-init logs and data')
-    if sysv_args and sysv_args[0] == 'analyze':
-        # Only load this parser if analyze is specified to avoid file load cost
-        # FIXME put this under 'devel' subcommand (coming in next branch)
-        from cloudinit.analyze.__main__ import get_parser as analyze_parser
-        # Construct analyze subcommand parser
-        analyze_parser(parser_analyze)
+
+    parser_devel = subparsers.add_parser(
+        'devel', help='Run development tools')
+
+    if sysv_args:
+        # Only load subparsers if subcommand is specified to avoid load cost
+        if sysv_args[0] == 'analyze':
+            from cloudinit.analyze.__main__ import get_parser as analyze_parser
+            # Construct analyze subcommand parser
+            analyze_parser(parser_analyze)
+        if sysv_args[0] == 'devel':
+            from cloudinit.cmd.devel.parser import get_parser as devel_parser
+            # Construct devel subcommand parser
+            devel_parser(parser_devel)
 
     args = parser.parse_args(args=sysv_args)
 
diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py
index dfa8cb3d..7c3ccd41 100644
--- a/cloudinit/config/cc_runcmd.py
+++ b/cloudinit/config/cc_runcmd.py
@@ -6,41 +6,66 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""
-Runcmd
-------
-**Summary:** run commands
+"""Runcmd: run arbitrary commands at rc.local with output to the console"""
 
-Run arbitrary commands at a rc.local like level with output to the console.
-Each item can be either a list or a string. If the item is a list, it will be
-properly executed as if passed to ``execve()`` (with the first arg as the
-command). If the item is a string, it will be written to a file and interpreted
-using ``sh``.
-
-.. note::
-    all commands must be proper yaml, so you have to quote any characters yaml
-    would eat (':' can be problematic)
-
-**Internal name:** ``cc_runcmd``
-
-**Module frequency:** per instance
+from cloudinit.config.schema import validate_cloudconfig_schema
+from cloudinit.settings import PER_INSTANCE
+from cloudinit import util
 
-**Supported distros:** all
+import os
+from textwrap import dedent
 
-**Config keys**::
 
-    runcmd:
-        - [ ls, -l, / ]
-        - [ sh, -xc, "echo $(date) ': hello world!'" ]
-        - [ sh, -c, echo "=========hello world'=========" ]
-        - ls -l /root
-        - [ wget, "http://example.org", -O, /tmp/index.html ]
-"""
+# The schema definition for each cloud-config module is a strict contract for
+# describing supported configuration parameters for each cloud-config section.
+# It allows cloud-config to validate and alert users to invalid or ignored
+# configuration options before actually attempting to deploy with said
+# configuration.
 
+distros = ['all']
 
-import os
+schema = {
+    'id': 'cc_runcmd',
+    'name': 'Runcmd',
+    'title': 'Run arbitrary commands',
+    'description': dedent("""\
+        Run arbitrary commands at a rc.local like level with output to the
+        console. Each item can be either a list or a string. If the item is a
+        list, it will be properly executed as if passed to ``execve()`` (with
+        the first arg as the command). If the item is a string, it will be
+        written to a file and interpreted
+        using ``sh``.
 
-from cloudinit import util
+        .. note::
+        all commands must be proper yaml, so you have to quote any characters
+        yaml would eat (':' can be problematic)"""),
+    'distros': distros,
+    'examples': [dedent("""\
+        runcmd:
+            - [ ls, -l, / ]
+            - [ sh, -xc, "echo $(date) ': hello world!'" ]
+            - [ sh, -c, echo "=========hello world'=========" ]
+            - ls -l /root
+            - [ wget, "http://example.org", -O, /tmp/index.html ]
+    """)],
+    'frequency': PER_INSTANCE,
+    'type': 'object',
+    'properties': {
+        'runcmd': {
+            'type': 'array',
+            'items': {
+                'oneOf': [
+                    {'type': 'array', 'items': {'type': 'string'}},
+                    {'type': 'string'}]
+            },
+            'additionalItems': False,  # Reject items of non-string non-list
+            'additionalProperties': False,
+            'minItems': 1,
+            'required': [],
+            'uniqueItems': True
+        }
+    }
+}
 
 
 def handle(name, cfg, cloud, log, _args):
@@ -49,6 +74,7 @@ def handle(name, cfg, cloud, log, _args):
                    " no 'runcmd' key in configuration"), name)
         return
 
+    validate_cloudconfig_schema(cfg, schema)
     out_fn = os.path.join(cloud.get_ipath('scripts'), "runcmd")
     cmd = cfg["runcmd"]
     try:
diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py
index 6400f005..73dd5c2e 100644
--- a/cloudinit/config/schema.py
+++ b/cloudinit/config/schema.py
@@ -3,11 +3,14 @@
 
 from __future__ import print_function
 
-from cloudinit.util import read_file_or_url
+from cloudinit import importer
+from cloudinit.util import find_modules, read_file_or_url
 
 import argparse
+from collections import defaultdict
 import logging
 import os
+import re
 import sys
 import yaml
 
@@ -15,7 +18,7 @@ SCHEMA_UNDEFINED = b'UNDEFINED'
 CLOUD_CONFIG_HEADER = b'#cloud-config'
 SCHEMA_DOC_TMPL = """
 {name}
----
+{title_underbar}
 **Summary:** {title}
 
 {description}
@@ -83,11 +86,49 @@ def validate_cloudconfig_schema(config, schema, strict=False):
             logging.warning('Invalid config:\n%s', '\n'.join(messages))
 
 
-def validate_cloudconfig_file(config_path, schema):
+def annotated_cloudconfig_file(cloudconfig, original_content, schema_errors):
+    """Return contents of the cloud-config file annotated with schema errors.
+
+    @param cloudconfig: YAML-loaded object from the original_content.
+    @param original_content: The contents of a cloud-config file
+    @param schema_errors: List of tuples from a JSONSchemaValidationError. The
+        tuples consist of (schemapath, error_message).
+    """
+    if not schema_errors:
+        return original_content
+    schemapaths = _schemapath_for_cloudconfig(cloudconfig, original_content)
+    errors_by_line = defaultdict(list)
+    error_count = 1
+    error_footer = []
+    annotated_content = []
+    for path, msg in schema_errors:
+        errors_by_line[schemapaths[path]].append(msg)
+        error_footer.append('# E{0}: {1}'.format(error_count, msg))
+        error_count += 1
+    lines = original_content.decode().split('\n')
+    error_count = 1
+    for line_number, line in enumerate(lines):
+        errors = errors_by_line[line_number + 1]
+        if errors:
+            error_label = ','.join(
+                ['E{0}'.format(count + error_count)
+                 for count in range(0, len(errors))])
+            error_count += len(errors)
+            annotated_content.append(line + '\t\t# ' + error_label)
+        else:
+            annotated_content.append(line)
+    annotated_content.append(
+        '# Errors: -------------\n{0}\n\n'.format('\n'.join(error_footer)))
+    return '\n'.join(annotated_content)
+
+
+def validate_cloudconfig_file(config_path, schema, annotate=False):
     """Validate cloudconfig file adheres to a specific jsonschema.
 
     @param config_path: Path to the yaml cloud-config file to parse.
     @param schema: Dict describing a valid jsonschema to validate against.
+    @param annotate: Boolean set True to print original config file with error
+        annotations on the offending lines.
 
     @raises SchemaValidationError containing any of schema_errors encountered.
     @raises RuntimeError when config_path does not exist.
@@ -108,8 +149,64 @@ def validate_cloudconfig_file(config_path, schema):
             ('format', 'File {0} is not valid yaml. {1}'.format(
                 config_path, str(e))),)
         raise SchemaValidationError(errors)
-    validate_cloudconfig_schema(
-        cloudconfig, schema, strict=True)
+
+    try:
+        validate_cloudconfig_schema(
+            cloudconfig, schema, strict=True)
+    except SchemaValidationError as e:
+        if annotate:
+            print(annotated_cloudconfig_file(
+                cloudconfig, content, e.schema_errors))
+        raise
+
+
+def _schemapath_for_cloudconfig(config, original_content):
+    """Return a dictionary mapping schemapath to original_content line number.
+
+    @param config: The yaml.loaded config dictionary of a cloud-config file.
+    @param original_content: The simple file content of the cloud-config file
+    """
+    # FIXME Doesn't handle multi-line lists or multi-line strings
+    content_lines = original_content.decode().split('\n')
+    schema_line_numbers = {}
+    list_index = 0
+    RE_YAML_INDENT = r'^(\s*)'
+    scopes = []
+    for line_number, line in enumerate(content_lines):
+        indent_depth = len(re.match(RE_YAML_INDENT, line).groups()[0])
+        line = line.strip()
+        if not line or line.startswith('#'):
+            continue
+        if scopes:
+            previous_depth, path_prefix = scopes[-1]
+        else:
+            previous_depth = -1
+            path_prefix = ''
+        if line.startswith('- '):
+            key = str(list_index)
+            value = line[1:]
+            list_index += 1
+        else:
+            list_index = 0
+            key, value = line.split(':', 1)
+        while indent_depth <= previous_depth:
+            if scopes:
+                previous_depth, path_prefix = scopes.pop()
+            else:
+                previous_depth = -1
+                path_prefix = ''
+        if path_prefix:
+            key = path_prefix + '.' + key
+        scopes.append((indent_depth, key))
+        if value:
+            value = value.strip()
+            if value.startswith('['):
+                scopes.append((indent_depth + 2, key + '.0'))
+                for inner_list_index in range(0, len(yaml.safe_load(value))):
+                    list_key = key + '.' + str(inner_list_index)
+                    schema_line_numbers[list_key] = line_number + 1
+        schema_line_numbers[key] = line_number + 1
+    return schema_line_numbers
 
 
 def _get_property_type(property_dict):
@@ -117,9 +214,15 @@ def _get_property_type(property_dict):
     property_type = property_dict.get('type', SCHEMA_UNDEFINED)
     if isinstance(property_type, list):
         property_type = '/'.join(property_type)
-    item_type = property_dict.get('items', {}).get('type')
-    if item_type:
-        property_type = '{0} of {1}'.format(property_type, item_type)
+    items = property_dict.get('items', {})
+    sub_property_type = items.get('type', '')
+    # Collect each item type
+    for sub_item in items.get('oneOf', {}):
+        if sub_property_type:
+            sub_property_type += '/'
+        sub_property_type += '(' + _get_property_type(sub_item) + ')'
+    if sub_property_type:
+        return '{0} of {1}'.format(property_type, sub_property_type)
     return property_type
 
 
@@ -148,9 +251,12 @@ def _get_schema_examples(schema, prefix=''):
         return ''
     rst_content = '\n**Examples**::\n\n'
     for example in examples:
-        example_yaml = yaml.dump(example, default_flow_style=False)
+        if isinstance(example, str):
+            example_content = example
+        else:
+            example_content = yaml.dump(example, default_flow_style=False)
         # Python2.6 is missing textwrapper.indent
-        lines = example_yaml.split('\n')
+        lines = example_content.split('\n')
         indented_lines = ['    {0}'.format(line) for line in lines]
         rst_content += '\n'.join(indented_lines)
     return rst_content
@@ -165,58 +271,83 @@ def get_schema_doc(schema):
     schema['property_doc'] = _get_property_doc(schema)
     schema['examples'] = _get_schema_examples(schema)
     schema['distros'] = ', '.join(schema['distros'])
+    # Need an underbar of the same length as the name
+    schema['title_underbar'] = re.sub(r'.', '-', schema['name'])
     return SCHEMA_DOC_TMPL.format(**schema)
 
 
-def get_schema(section_key=None):
-    """Return a dict of jsonschema defined in any cc_* module.
+FULL_SCHEMA = None
 
-    @param: section_key: Optionally limit schema to a specific top-level key.
-    """
-    # TODO use util.find_modules in subsequent branch
-    from cloudinit.config.cc_ntp import schema
-    return schema
+
+def get_schema():
+    """Return jsonschema coalesced from all cc_* cloud-config module."""
+    global FULL_SCHEMA
+    if FULL_SCHEMA:
+        return FULL_SCHEMA
+    full_schema = {
+        '$schema': 'http://json-schema.org/draft-04/schema#',
+        'id': 'cloud-config-schema', 'allOf': []}
+
+    configs_dir = os.path.dirname(os.path.abspath(__file__))
+    potential_handlers = find_modules(configs_dir)
+    for (fname, mod_name) in potential_handlers.items():
+        mod_locs, looked_locs = importer.find_module(
+            mod_name, ['cloudinit.config'], ['schema'])
+        if mod_locs:
+            mod = importer.import_module(mod_locs[0])
+            full_schema['allOf'].append(mod.schema)
+    FULL_SCHEMA = full_schema
+    return full_schema
 
 
 def error(message):
     print(message, file=sys.stderr)
-    return 1
+    sys.exit(1)
 
 
-def get_parser():
+def get_parser(parser=None):
     """Return a parser for supported cmdline arguments."""
-    parser = argparse.ArgumentParser()
+    if not parser:
+        parser = argparse.ArgumentParser(
+            prog='cloudconfig-schema',
+            description='Validate cloud-config files or document schema')
     parser.add_argument('-c', '--config-file',
                         help='Path of the cloud-config yaml file to validate')
     parser.add_argument('-d', '--doc', action="store_true", default=False,
                         help='Print schema documentation')
-    parser.add_argument('-k', '--key',
-                        help='Limit validation or docs to a section key')
+    parser.add_argument('--annotate', action="store_true", default=False,
+                        help='Annotate existing cloud-config file with errors')
     return parser
 
 
-def main():
-    """Tool to validate schema of a cloud-config file or print schema docs."""
-    parser = get_parser()
-    args = parser.parse_args()
+def handle_schema_args(name, args):
+    """Handle provided schema args and perform the appropriate actions."""
     exclusive_args = [args.config_file, args.doc]
     if not any(exclusive_args) or all(exclusive_args):
-        return error('Expected either --config-file argument or --doc')
-
-    schema = get_schema()
+        error('Expected either --config-file argument or --doc')
+    full_schema = get_schema()
     if args.config_file:
         try:
-            validate_cloudconfig_file(args.config_file, schema)
+            validate_cloudconfig_file(
+                args.config_file, full_schema, args.annotate)
         except (SchemaValidationError, RuntimeError) as e:
-            return error(str(e))
-        print("Valid cloud-config file {0}".format(args.config_file))
+            if not args.annotate:
+                error(str(e))
+        else:
+            print("Valid cloud-config file {0}".format(args.config_file))
     if args.doc:
-        print(get_schema_doc(schema))
+        for subschema in full_schema['allOf']:
+            print(get_schema_doc(subschema))
+
+
+def main():
+    """Tool to validate schema of a cloud-config file or print schema docs."""
+    parser = get_parser()
+    handle_schema_args('cloudconfig-schema', parser.parse_args())
     return 0
 
 
 if __name__ == '__main__':
     sys.exit(main())
 
-
 # vi: ts=4 expandtab