ntp: Add schema definition and passive schema validation.

cloud-config files are very flexible and permissive.
This adds a jsonsschema definition to the cc_ntp module and validation
functions in cloudinit/config/schema which will log warnings about
invalid configuration values in the ntp section.

A cmdline tools/cloudconfig-schema is added which can be used in our dev
environments to quickly attempt to exercise the ntp schema.
It is also exposed as a main in cloudinit.config.schema.
(python3 -m cloudinit.config.schema)

LP: #1692916

2 files changed, 289 insertions, 2 deletions

diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py
index 5cc54536..31ed64e3 100644
--- a/cloudinit/config/cc_ntp.py
+++ b/cloudinit/config/cc_ntp.py
@@ -36,6 +36,7 @@ servers or pools are provided, 4 pools will be used in the format
             - 192.168.23.2
 """
 
+from cloudinit.config.schema import validate_cloudconfig_schema
 from cloudinit import log as logging
 from cloudinit.settings import PER_INSTANCE
 from cloudinit import templater
@@ -43,6 +44,7 @@ from cloudinit import type_utils
 from cloudinit import util
 
 import os
+from textwrap import dedent
 
 LOG = logging.getLogger(__name__)
 
@@ -52,21 +54,84 @@ NR_POOL_SERVERS = 4
 distros = ['centos', 'debian', 'fedora', 'opensuse', 'ubuntu']
 
 
+# The schema definition for each cloud-config module is a strict contract for
+# describing supported configuration parameters for each cloud-config section.
+# It allows cloud-config to validate and alert users to invalid or ignored
+# configuration options before actually attempting to deploy with said
+# configuration.
+
+schema = {
+    'id': 'cc_ntp',
+    'name': 'NTP',
+    'title': 'enable and configure ntp',
+    'description': dedent("""\
+        Handle ntp configuration. If ntp is not installed on the system and
+        ntp configuration is specified, ntp will be installed. If there is a
+        default ntp config file in the image or one is present in the
+        distro's ntp package, it will be copied to ``/etc/ntp.conf.dist``
+        before any changes are made. A list of ntp pools and ntp servers can
+        be provided under the ``ntp`` config key. If no ntp ``servers`` or
+        ``pools`` are provided, 4 pools will be used in the format
+        ``{0-3}.{distro}.pool.ntp.org``."""),
+    'distros': distros,
+    'examples': [
+        {'ntp': {'pools': ['0.company.pool.ntp.org', '1.company.pool.ntp.org',
+                           'ntp.myorg.org'],
+                 'servers': ['my.ntp.server.local', 'ntp.ubuntu.com',
+                             '192.168.23.2']}}],
+    'frequency': PER_INSTANCE,
+    'type': 'object',
+    'properties': {
+        'ntp': {
+            'type': ['object', 'null'],
+            'properties': {
+                'pools': {
+                    'type': 'array',
+                    'items': {
+                        'type': 'string',
+                        'format': 'hostname'
+                    },
+                    'uniqueItems': True,
+                    'description': dedent("""\
+                        List of ntp pools. If both pools and servers are
+                         empty, 4 default pool servers will be provided of
+                         the format ``{0-3}.{distro}.pool.ntp.org``.""")
+                },
+                'servers': {
+                    'type': 'array',
+                    'items': {
+                        'type': 'string',
+                        'format': 'hostname'
+                    },
+                    'uniqueItems': True,
+                    'description': dedent("""\
+                        List of ntp servers. If both pools and servers are
+                         empty, 4 default pool servers will be provided with
+                         the format ``{0-3}.{distro}.pool.ntp.org``.""")
+                }
+            },
+            'required': [],
+            'additionalProperties': False
+        }
+    }
+}
+
+
 def handle(name, cfg, cloud, log, _args):
     """Enable and configure ntp."""
-
     if 'ntp' not in cfg:
         LOG.debug(
             "Skipping module named %s, not present or disabled by cfg", name)
         return
-
     ntp_cfg = cfg.get('ntp', {})
 
+    # TODO drop this when validate_cloudconfig_schema is strict=True
     if not isinstance(ntp_cfg, (dict)):
         raise RuntimeError(("'ntp' key existed in config,"
                             " but not a dictionary type,"
                             " is a %s %instead"), type_utils.obj_name(ntp_cfg))
 
+    validate_cloudconfig_schema(cfg, schema)
     rename_ntp_conf()
     # ensure when ntp is installed it has a configuration file
     # to use instead of starting up with packaged defaults
diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py
new file mode 100644
index 00000000..6400f005
--- /dev/null
+++ b/cloudinit/config/schema.py
@@ -0,0 +1,222 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+"""schema.py: Set of module functions for processing cloud-config schema."""
+
+from __future__ import print_function
+
+from cloudinit.util import read_file_or_url
+
+import argparse
+import logging
+import os
+import sys
+import yaml
+
+SCHEMA_UNDEFINED = b'UNDEFINED'
+CLOUD_CONFIG_HEADER = b'#cloud-config'
+SCHEMA_DOC_TMPL = """
+{name}
+---
+**Summary:** {title}
+
+{description}
+
+**Internal name:** ``{id}``
+
+**Module frequency:** {frequency}
+
+**Supported distros:** {distros}
+
+**Config schema**:
+{property_doc}
+{examples}
+"""
+SCHEMA_PROPERTY_TMPL = '{prefix}**{prop_name}:** ({type}) {description}'
+
+
+class SchemaValidationError(ValueError):
+    """Raised when validating a cloud-config file against a schema."""
+
+    def __init__(self, schema_errors=()):
+        """Init the exception an n-tuple of schema errors.
+
+        @param schema_errors: An n-tuple of the format:
+            ((flat.config.key, msg),)
+        """
+        self.schema_errors = schema_errors
+        error_messages = [
+            '{0}: {1}'.format(config_key, message)
+            for config_key, message in schema_errors]
+        message = "Cloud config schema errors: {0}".format(
+            ', '.join(error_messages))
+        super(SchemaValidationError, self).__init__(message)
+
+
+def validate_cloudconfig_schema(config, schema, strict=False):
+    """Validate provided config meets the schema definition.
+
+    @param config: Dict of cloud configuration settings validated against
+        schema.
+    @param schema: jsonschema dict describing the supported schema definition
+       for the cloud config module (config.cc_*).
+    @param strict: Boolean, when True raise SchemaValidationErrors instead of
+       logging warnings.
+
+    @raises: SchemaValidationError when provided config does not validate
+        against the provided schema.
+    """
+    try:
+        from jsonschema import Draft4Validator, FormatChecker
+    except ImportError:
+        logging.warning(
+            'Ignoring schema validation. python-jsonschema is not present')
+        return
+    validator = Draft4Validator(schema, format_checker=FormatChecker())
+    errors = ()
+    for error in sorted(validator.iter_errors(config), key=lambda e: e.path):
+        path = '.'.join([str(p) for p in error.path])
+        errors += ((path, error.message),)
+    if errors:
+        if strict:
+            raise SchemaValidationError(errors)
+        else:
+            messages = ['{0}: {1}'.format(k, msg) for k, msg in errors]
+            logging.warning('Invalid config:\n%s', '\n'.join(messages))
+
+
+def validate_cloudconfig_file(config_path, schema):
+    """Validate cloudconfig file adheres to a specific jsonschema.
+
+    @param config_path: Path to the yaml cloud-config file to parse.
+    @param schema: Dict describing a valid jsonschema to validate against.
+
+    @raises SchemaValidationError containing any of schema_errors encountered.
+    @raises RuntimeError when config_path does not exist.
+    """
+    if not os.path.exists(config_path):
+        raise RuntimeError('Configfile {0} does not exist'.format(config_path))
+    content = read_file_or_url('file://{0}'.format(config_path)).contents
+    if not content.startswith(CLOUD_CONFIG_HEADER):
+        errors = (
+            ('header', 'File {0} needs to begin with "{1}"'.format(
+                config_path, CLOUD_CONFIG_HEADER.decode())),)
+        raise SchemaValidationError(errors)
+
+    try:
+        cloudconfig = yaml.safe_load(content)
+    except yaml.parser.ParserError as e:
+        errors = (
+            ('format', 'File {0} is not valid yaml. {1}'.format(
+                config_path, str(e))),)
+        raise SchemaValidationError(errors)
+    validate_cloudconfig_schema(
+        cloudconfig, schema, strict=True)
+
+
+def _get_property_type(property_dict):
+    """Return a string representing a property type from a given jsonschema."""
+    property_type = property_dict.get('type', SCHEMA_UNDEFINED)
+    if isinstance(property_type, list):
+        property_type = '/'.join(property_type)
+    item_type = property_dict.get('items', {}).get('type')
+    if item_type:
+        property_type = '{0} of {1}'.format(property_type, item_type)
+    return property_type
+
+
+def _get_property_doc(schema, prefix='    '):
+    """Return restructured text describing the supported schema properties."""
+    new_prefix = prefix + '    '
+    properties = []
+    for prop_key, prop_config in schema.get('properties', {}).items():
+        # Define prop_name and dscription for SCHEMA_PROPERTY_TMPL
+        description = prop_config.get('description', '')
+        properties.append(SCHEMA_PROPERTY_TMPL.format(
+            prefix=prefix,
+            prop_name=prop_key,
+            type=_get_property_type(prop_config),
+            description=description.replace('\n', '')))
+        if 'properties' in prop_config:
+            properties.append(
+                _get_property_doc(prop_config, prefix=new_prefix))
+    return '\n\n'.join(properties)
+
+
+def _get_schema_examples(schema, prefix=''):
+    """Return restructured text describing the schema examples if present."""
+    examples = schema.get('examples')
+    if not examples:
+        return ''
+    rst_content = '\n**Examples**::\n\n'
+    for example in examples:
+        example_yaml = yaml.dump(example, default_flow_style=False)
+        # Python2.6 is missing textwrapper.indent
+        lines = example_yaml.split('\n')
+        indented_lines = ['    {0}'.format(line) for line in lines]
+        rst_content += '\n'.join(indented_lines)
+    return rst_content
+
+
+def get_schema_doc(schema):
+    """Return reStructured text rendering the provided jsonschema.
+
+    @param schema: Dict of jsonschema to render.
+    @raise KeyError: If schema lacks an expected key.
+    """
+    schema['property_doc'] = _get_property_doc(schema)
+    schema['examples'] = _get_schema_examples(schema)
+    schema['distros'] = ', '.join(schema['distros'])
+    return SCHEMA_DOC_TMPL.format(**schema)
+
+
+def get_schema(section_key=None):
+    """Return a dict of jsonschema defined in any cc_* module.
+
+    @param: section_key: Optionally limit schema to a specific top-level key.
+    """
+    # TODO use util.find_modules in subsequent branch
+    from cloudinit.config.cc_ntp import schema
+    return schema
+
+
+def error(message):
+    print(message, file=sys.stderr)
+    return 1
+
+
+def get_parser():
+    """Return a parser for supported cmdline arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', '--config-file',
+                        help='Path of the cloud-config yaml file to validate')
+    parser.add_argument('-d', '--doc', action="store_true", default=False,
+                        help='Print schema documentation')
+    parser.add_argument('-k', '--key',
+                        help='Limit validation or docs to a section key')
+    return parser
+
+
+def main():
+    """Tool to validate schema of a cloud-config file or print schema docs."""
+    parser = get_parser()
+    args = parser.parse_args()
+    exclusive_args = [args.config_file, args.doc]
+    if not any(exclusive_args) or all(exclusive_args):
+        return error('Expected either --config-file argument or --doc')
+
+    schema = get_schema()
+    if args.config_file:
+        try:
+            validate_cloudconfig_file(args.config_file, schema)
+        except (SchemaValidationError, RuntimeError) as e:
+            return error(str(e))
+        print("Valid cloud-config file {0}".format(args.config_file))
+    if args.doc:
+        print(get_schema_doc(schema))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())
+
+
+# vi: ts=4 expandtab