schema and docs: Add jsonschema to resizefs and bootcmd modules

Add schema definitions to both cc_resizefs and cc_bootcmd modules. Extend
schema.py to parse and document enumerated json types. Schema definitions
are used to generate module documention and log warnings for schema
infractions.

This branch also does the following:
  - drops vestigial 'resize_rootfs_tmp' option from cc_resizefs. That
    option only created the specified directory and didn't make use of
that directory for any resize operations.
  - Drop yaml.dumps calls from schema documentation generation to avoid
    yaml import costs on module load
  - Add __doc__ = get_schema_doc(schema) definitions it each module to
    supplement python help() calls for cc_runcmd, cc_bootcmd, cc_ntp and
    cc_resizefs
  - Add a SCHEMA_EXAMPLES_SPACER_TEMPLATE string to docs for modules which
    contain more than one example

5 files changed, 171 insertions, 137 deletions

diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py
index 9c0476af..233da1ef 100644
--- a/cloudinit/config/cc_bootcmd.py
+++ b/cloudinit/config/cc_bootcmd.py
@@ -3,45 +3,73 @@
 #
 # Author: Scott Moser <scott.moser@canonical.com>
 # Author: Juerg Haefliger <juerg.haefliger@hp.com>
+# Author: Chad Smith <chad.smith@canonical.com>
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""
-Bootcmd
--------
-**Summary:** run commands early in boot process
-
-This module runs arbitrary commands very early in the boot process,
-only slightly after a boothook would run. This is very similar to a
-boothook, but more user friendly. The environment variable ``INSTANCE_ID``
-will be set to the current instance id for all run commands. Commands can be
-specified either as lists or strings. For invocation details, see ``runcmd``.
-
-.. note::
-    bootcmd should only be used for things that could not be done later in the
-    boot process.
-
-**Internal name:** ``cc_bootcmd``
-
-**Module frequency:** per always
-
-**Supported distros:** all
-
-**Config keys**::
-
-    bootcmd:
-        - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts
-        - [ cloud-init-per, once, mymkfs, mkfs, /dev/vdb ]
-"""
+"""Bootcmd: run arbitrary commands early in the boot process."""
 
 import os
+from textwrap import dedent
 
+from cloudinit.config.schema import (
+    get_schema_doc, validate_cloudconfig_schema)
 from cloudinit.settings import PER_ALWAYS
 from cloudinit import temp_utils
 from cloudinit import util
 
 frequency = PER_ALWAYS
 
+# The schema definition for each cloud-config module is a strict contract for
+# describing supported configuration parameters for each cloud-config section.
+# It allows cloud-config to validate and alert users to invalid or ignored
+# configuration options before actually attempting to deploy with said
+# configuration.
+
+distros = ['all']
+
+schema = {
+    'id': 'cc_bootcmd',
+    'name': 'Bootcmd',
+    'title': 'Run arbitrary commands early in the boot process',
+    'description': dedent("""\
+        This module runs arbitrary commands very early in the boot process,
+        only slightly after a boothook would run. This is very similar to a
+        boothook, but more user friendly. The environment variable
+        ``INSTANCE_ID`` will be set to the current instance id for all run
+        commands. Commands can be specified either as lists or strings. For
+        invocation details, see ``runcmd``.
+
+        .. note::
+            bootcmd should only be used for things that could not be done later
+            in the boot process."""),
+    'distros': distros,
+    'examples': [dedent("""\
+        bootcmd:
+            - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts
+            - [ cloud-init-per, once, mymkfs, mkfs, /dev/vdb ]
+    """)],
+    'frequency': PER_ALWAYS,
+    'type': 'object',
+    'properties': {
+        'bootcmd': {
+            'type': 'array',
+            'items': {
+                'oneOf': [
+                    {'type': 'array', 'items': {'type': 'string'}},
+                    {'type': 'string'}]
+            },
+            'additionalItems': False,  # Reject items of non-string non-list
+            'additionalProperties': False,
+            'minItems': 1,
+            'required': [],
+            'uniqueItems': True
+        }
+    }
+}
+
+__doc__ = get_schema_doc(schema)  # Supplement python help()
+
 
 def handle(name, cfg, cloud, log, _args):
 
@@ -50,13 +78,14 @@ def handle(name, cfg, cloud, log, _args):
                    " no 'bootcmd' key in configuration"), name)
         return
 
+    validate_cloudconfig_schema(cfg, schema)
     with temp_utils.ExtendedTemporaryFile(suffix=".sh") as tmpf:
         try:
             content = util.shellify(cfg["bootcmd"])
             tmpf.write(util.encode_text(content))
             tmpf.flush()
-        except Exception:
-            util.logexc(log, "Failed to shellify bootcmd")
+        except Exception as e:
+            util.logexc(log, "Failed to shellify bootcmd: %s", str(e))
             raise
 
         try:
diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py
index a02b4bf1..15ae1ecd 100644
--- a/cloudinit/config/cc_ntp.py
+++ b/cloudinit/config/cc_ntp.py
@@ -4,39 +4,10 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""
-NTP
----
-**Summary:** enable and configure ntp
-
-Handle ntp configuration. If ntp is not installed on the system and ntp
-configuration is specified, ntp will be installed. If there is a default ntp
-config file in the image or one is present in the distro's ntp package, it will
-be copied to ``/etc/ntp.conf.dist`` before any changes are made. A list of ntp
-pools and ntp servers can be provided under the ``ntp`` config key. If no ntp
-servers or pools are provided, 4 pools will be used in the format
-``{0-3}.{distro}.pool.ntp.org``.
-
-**Internal name:** ``cc_ntp``
-
-**Module frequency:** per instance
-
-**Supported distros:** centos, debian, fedora, opensuse, ubuntu
-
-**Config keys**::
-
-    ntp:
-        pools:
-            - 0.company.pool.ntp.org
-            - 1.company.pool.ntp.org
-            - ntp.myorg.org
-        servers:
-            - my.ntp.server.local
-            - ntp.ubuntu.com
-            - 192.168.23.2
-"""
+"""NTP: enable and configure ntp"""
 
-from cloudinit.config.schema import validate_cloudconfig_schema
+from cloudinit.config.schema import (
+    get_schema_doc, validate_cloudconfig_schema)
 from cloudinit import log as logging
 from cloudinit.settings import PER_INSTANCE
 from cloudinit import templater
@@ -76,10 +47,13 @@ schema = {
         ``{0-3}.{distro}.pool.ntp.org``."""),
     'distros': distros,
     'examples': [
-        {'ntp': {'pools': ['0.company.pool.ntp.org', '1.company.pool.ntp.org',
-                           'ntp.myorg.org'],
-                 'servers': ['my.ntp.server.local', 'ntp.ubuntu.com',
-                             '192.168.23.2']}}],
+        dedent("""\
+        ntp:
+          pools: [0.int.pool.ntp.org, 1.int.pool.ntp.org, ntp.myorg.org]
+          servers:
+            - ntp.server.local
+            - ntp.ubuntu.com
+            - 192.168.23.2""")],
     'frequency': PER_INSTANCE,
     'type': 'object',
     'properties': {
@@ -117,6 +91,8 @@ schema = {
     }
 }
 
+__doc__ = get_schema_doc(schema)  # Supplement python help()
+
 
 def handle(name, cfg, cloud, log, _args):
     """Enable and configure ntp."""
diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py
index ceee952b..f14d3836 100644
--- a/cloudinit/config/cc_resizefs.py
+++ b/cloudinit/config/cc_resizefs.py
@@ -6,31 +6,8 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
-"""
-Resizefs
---------
-**Summary:** resize filesystem
+"""Resizefs: cloud-config module which resizes the filesystem"""
 
-Resize a filesystem to use all avaliable space on partition. This module is
-useful along with ``cc_growpart`` and will ensure that if the root partition
-has been resized the root filesystem will be resized along with it. By default,
-``cc_resizefs`` will resize the root partition and will block the boot process
-while the resize command is running. Optionally, the resize operation can be
-performed in the background while cloud-init continues running modules. This
-can be enabled by setting ``resize_rootfs`` to ``true``. This module can be
-disabled altogether by setting ``resize_rootfs`` to ``false``.
-
-**Internal name:** ``cc_resizefs``
-
-**Module frequency:** per always
-
-**Supported distros:** all
-
-**Config keys**::
-
-    resize_rootfs: <true/false/"noblock">
-    resize_rootfs_tmp: <directory>
-"""
 
 import errno
 import getopt
@@ -38,11 +15,47 @@ import os
 import re
 import shlex
 import stat
+from textwrap import dedent
 
+from cloudinit.config.schema import (
+    get_schema_doc, validate_cloudconfig_schema)
 from cloudinit.settings import PER_ALWAYS
 from cloudinit import util
 
+NOBLOCK = "noblock"
+
 frequency = PER_ALWAYS
+distros = ['all']
+
+schema = {
+    'id': 'cc_resizefs',
+    'name': 'Resizefs',
+    'title': 'Resize filesystem',
+    'description': dedent("""\
+        Resize a filesystem to use all avaliable space on partition. This
+        module is useful along with ``cc_growpart`` and will ensure that if the
+        root partition has been resized the root filesystem will be resized
+        along with it. By default, ``cc_resizefs`` will resize the root
+        partition and will block the boot process while the resize command is
+        running. Optionally, the resize operation can be performed in the
+        background while cloud-init continues running modules. This can be
+        enabled by setting ``resize_rootfs`` to ``true``. This module can be
+        disabled altogether by setting ``resize_rootfs`` to ``false``."""),
+    'distros': distros,
+    'examples': [
+        'resize_rootfs: false  # disable root filesystem resize operation'],
+    'frequency': PER_ALWAYS,
+    'type': 'object',
+    'properties': {
+        'resize_rootfs': {
+            'enum': [True, False, NOBLOCK],
+            'description': dedent("""\
+                Whether to resize the root partition. Default: 'true'""")
+        }
+    }
+}
+
+__doc__ = get_schema_doc(schema)  # Supplement python help()
 
 
 def _resize_btrfs(mount_point, devpth):
@@ -131,8 +144,6 @@ RESIZE_FS_PRECHECK_CMDS = {
     'ufs': _can_skip_resize_ufs
 }
 
-NOBLOCK = "noblock"
-
 
 def rootdev_from_cmdline(cmdline):
     found = None
@@ -161,71 +172,81 @@ def can_skip_resize(fs_type, resize_what, devpth):
     return False
 
 
-def handle(name, cfg, _cloud, log, args):
-    if len(args) != 0:
-        resize_root = args[0]
-    else:
-        resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True)
-
-    if not util.translate_bool(resize_root, addons=[NOBLOCK]):
-        log.debug("Skipping module named %s, resizing disabled", name)
-        return
-
-    # TODO(harlowja) is the directory ok to be used??
-    resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run")
-    util.ensure_dir(resize_root_d)
+def is_device_path_writable_block(devpath, info, log):
+    """Return True if devpath is a writable block device.
 
-    # TODO(harlowja): allow what is to be resized to be configurable??
-    resize_what = "/"
-    result = util.get_mount_info(resize_what, log)
-    if not result:
-        log.warn("Could not determine filesystem type of %s", resize_what)
-        return
-
-    (devpth, fs_type, mount_point) = result
-
-    info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what)
-    log.debug("resize_info: %s" % info)
+    @param devpath: Path to the root device we want to resize.
+    @param info: String representing information about the requested device.
+    @param log: Logger to which logs will be added upon error.
 
+    @returns Boolean True if block device is writable
+    """
     container = util.is_container()
 
     # Ensure the path is a block device.
-    if (devpth == "/dev/root" and not os.path.exists(devpth) and
+    if (devpath == "/dev/root" and not os.path.exists(devpath) and
             not container):
-        devpth = util.rootdev_from_cmdline(util.get_cmdline())
-        if devpth is None:
+        devpath = util.rootdev_from_cmdline(util.get_cmdline())
+        if devpath is None:
             log.warn("Unable to find device '/dev/root'")
-            return
-        log.debug("Converted /dev/root to '%s' per kernel cmdline", devpth)
+            return False
+        log.debug("Converted /dev/root to '%s' per kernel cmdline", devpath)
 
     try:
-        statret = os.stat(devpth)
+        statret = os.stat(devpath)
     except OSError as exc:
         if container and exc.errno == errno.ENOENT:
             log.debug("Device '%s' did not exist in container. "
-                      "cannot resize: %s", devpth, info)
+                      "cannot resize: %s", devpath, info)
         elif exc.errno == errno.ENOENT:
             log.warn("Device '%s' did not exist. cannot resize: %s",
-                     devpth, info)
+                     devpath, info)
         else:
             raise exc
-        return
+        return False
 
-    if not os.access(devpth, os.W_OK):
+    if not os.access(devpath, os.W_OK):
         if container:
             log.debug("'%s' not writable in container. cannot resize: %s",
-                      devpth, info)
+                      devpath, info)
         else:
-            log.warn("'%s' not writable. cannot resize: %s", devpth, info)
+            log.warn("'%s' not writable. cannot resize: %s", devpath, info)
         return
 
     if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode):
         if container:
             log.debug("device '%s' not a block device in container."
-                      " cannot resize: %s" % (devpth, info))
+                      " cannot resize: %s" % (devpath, info))
         else:
             log.warn("device '%s' not a block device. cannot resize: %s" %
-                     (devpth, info))
+                     (devpath, info))
+        return False
+    return True
+
+
+def handle(name, cfg, _cloud, log, args):
+    if len(args) != 0:
+        resize_root = args[0]
+    else:
+        resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True)
+    validate_cloudconfig_schema(cfg, schema)
+    if not util.translate_bool(resize_root, addons=[NOBLOCK]):
+        log.debug("Skipping module named %s, resizing disabled", name)
+        return
+
+    # TODO(harlowja): allow what is to be resized to be configurable??
+    resize_what = "/"
+    result = util.get_mount_info(resize_what, log)
+    if not result:
+        log.warn("Could not determine filesystem type of %s", resize_what)
+        return
+
+    (devpth, fs_type, mount_point) = result
+
+    info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what)
+    log.debug("resize_info: %s" % info)
+
+    if not is_device_path_writable_block(devpth, info, log):
         return
 
     resizer = None
diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py
index 7c3ccd41..7f995693 100644
--- a/cloudinit/config/cc_runcmd.py
+++ b/cloudinit/config/cc_runcmd.py
@@ -8,7 +8,8 @@
 
 """Runcmd: run arbitrary commands at rc.local with output to the console"""
 
-from cloudinit.config.schema import validate_cloudconfig_schema
+from cloudinit.config.schema import (
+    get_schema_doc, validate_cloudconfig_schema)
 from cloudinit.settings import PER_INSTANCE
 from cloudinit import util
 
@@ -67,6 +68,8 @@ schema = {
     }
 }
 
+__doc__ = get_schema_doc(schema)  # Supplement python help()
+
 
 def handle(name, cfg, cloud, log, _args):
     if "runcmd" not in cfg:
diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py
index 73dd5c2e..c17d973e 100644
--- a/cloudinit/config/schema.py
+++ b/cloudinit/config/schema.py
@@ -14,6 +14,7 @@ import re
 import sys
 import yaml
 
+_YAML_MAP = {True: 'true', False: 'false', None: 'null'}
 SCHEMA_UNDEFINED = b'UNDEFINED'
 CLOUD_CONFIG_HEADER = b'#cloud-config'
 SCHEMA_DOC_TMPL = """
@@ -34,6 +35,8 @@ SCHEMA_DOC_TMPL = """
 {examples}
 """
 SCHEMA_PROPERTY_TMPL = '{prefix}**{prop_name}:** ({type}) {description}'
+SCHEMA_EXAMPLES_HEADER = '\n**Examples**::\n\n'
+SCHEMA_EXAMPLES_SPACER_TEMPLATE = '\n    # --- Example{0} ---'
 
 
 class SchemaValidationError(ValueError):
@@ -212,6 +215,9 @@ def _schemapath_for_cloudconfig(config, original_content):
 def _get_property_type(property_dict):
     """Return a string representing a property type from a given jsonschema."""
     property_type = property_dict.get('type', SCHEMA_UNDEFINED)
+    if property_type == SCHEMA_UNDEFINED and property_dict.get('enum'):
+        property_type = [
+            str(_YAML_MAP.get(k, k)) for k in property_dict['enum']]
     if isinstance(property_type, list):
         property_type = '/'.join(property_type)
     items = property_dict.get('items', {})
@@ -249,15 +255,14 @@ def _get_schema_examples(schema, prefix=''):
     examples = schema.get('examples')
     if not examples:
         return ''
-    rst_content = '\n**Examples**::\n\n'
-    for example in examples:
-        if isinstance(example, str):
-            example_content = example
-        else:
-            example_content = yaml.dump(example, default_flow_style=False)
+    rst_content = SCHEMA_EXAMPLES_HEADER
+    for count, example in enumerate(examples):
         # Python2.6 is missing textwrapper.indent
-        lines = example_content.split('\n')
+        lines = example.split('\n')
         indented_lines = ['    {0}'.format(line) for line in lines]
+        if rst_content != SCHEMA_EXAMPLES_HEADER:
+            indented_lines.insert(
+                0, SCHEMA_EXAMPLES_SPACER_TEMPLATE.format(count + 1))
         rst_content += '\n'.join(indented_lines)
     return rst_content