schema cli: Add schema subcommand to cloud-init cli and cc_runcmd schema

This branch does a few things:
  - Add 'schema' subcommand to cloud-init CLI for validating
    cloud-config files against strict module jsonschema definitions
  - Add --annotate parameter to 'cloud-init schema' to annotate
    existing cloud-config file content with validation errors
  - Add jsonschema definition to cc_runcmd
  - Add unit test coverage for cc_runcmd
  - Update CLI capabilities documentation

This branch only imports development (and analyze) subparsers when the
specific subcommand is provided on the CLI to avoid adding costly unused
file imports during cloud-init system boot.

The schema command allows a person to quickly validate a cloud-config text
file against cloud-init's known module schemas to avoid costly roundtrips
deploying instances in their cloud of choice. As of this branch, only
cc_ntp and cc_runcmd cloud-config modules define schemas. Schema
validation will ignore all undefined config keys until all modules define
a strict schema.

To perform validation of runcmd and ntp sections of a cloud-config file:
$ cat > cloud.cfg <<EOF
runcmd: bogus
EOF
$ python -m cloudinit.cmd.main schema --config-file cloud.cfg

$ python -m cloudinit.cmd.main schema --config-file cloud.cfg \
  --annotate

Once jsonschema is defined for all ~55 cc modules, we will move this
schema subcommand up as a proper subcommand of the cloud-init CLI.

1 files changed, 108 insertions, 0 deletions

diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py
new file mode 100644
index 00000000..7880ee72
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_runcmd.py
@@ -0,0 +1,108 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit.config import cc_runcmd
+from cloudinit.sources import DataSourceNone
+from cloudinit import (distros, helpers, cloud, util)
+from ..helpers import FilesystemMockingTestCase, skipIf
+
+import logging
+import os
+import stat
+
+try:
+    import jsonschema
+    assert jsonschema  # avoid pyflakes error F401: import unused
+    _missing_jsonschema_dep = False
+except ImportError:
+    _missing_jsonschema_dep = True
+
+LOG = logging.getLogger(__name__)
+
+
+class TestRuncmd(FilesystemMockingTestCase):
+
+    with_logs = True
+
+    def setUp(self):
+        super(TestRuncmd, self).setUp()
+        self.subp = util.subp
+        self.new_root = self.tmp_dir()
+
+    def _get_cloud(self, distro):
+        self.patchUtils(self.new_root)
+        paths = helpers.Paths({'scripts': self.new_root})
+        cls = distros.fetch(distro)
+        mydist = cls(distro, {}, paths)
+        myds = DataSourceNone.DataSourceNone({}, mydist, paths)
+        paths.datasource = myds
+        return cloud.Cloud(myds, paths, {}, mydist, None)
+
+    def test_handler_skip_if_no_runcmd(self):
+        """When the provided config doesn't contain runcmd, skip it."""
+        cfg = {}
+        mycloud = self._get_cloud('ubuntu')
+        cc_runcmd.handle('notimportant', cfg, mycloud, LOG, None)
+        self.assertIn(
+            "Skipping module named notimportant, no 'runcmd' key",
+            self.logs.getvalue())
+
+    def test_handler_invalid_command_set(self):
+        """Commands which can't be converted to shell will raise errors."""
+        invalid_config = {'runcmd': 1}
+        cc = self._get_cloud('ubuntu')
+        cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, [])
+        self.assertIn(
+            'Failed to shellify 1 into file'
+            ' /var/lib/cloud/instances/iid-datasource-none/scripts/runcmd',
+            self.logs.getvalue())
+
+    @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
+    def test_handler_schema_validation_warns_non_array_type(self):
+        """Schema validation warns of non-array type for runcmd key.
+
+        Schema validation is not strict, so runcmd attempts to shellify the
+        invalid content.
+        """
+        invalid_config = {'runcmd': 1}
+        cc = self._get_cloud('ubuntu')
+        cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, [])
+        self.assertIn(
+            'Invalid config:\nruncmd: 1 is not of type \'array\'',
+            self.logs.getvalue())
+        self.assertIn('Failed to shellify', self.logs.getvalue())
+
+    @skipIf(_missing_jsonschema_dep, 'No python-jsonschema dependency')
+    def test_handler_schema_validation_warns_non_array_item_type(self):
+        """Schema validation warns of non-array or string runcmd items.
+
+        Schema validation is not strict, so runcmd attempts to shellify the
+        invalid content.
+        """
+        invalid_config = {
+            'runcmd': ['ls /', 20, ['wget', 'http://stuff/blah'], {'a': 'n'}]}
+        cc = self._get_cloud('ubuntu')
+        cc_runcmd.handle('cc_runcmd', invalid_config, cc, LOG, [])
+        expected_warnings = [
+            'runcmd.1: 20 is not valid under any of the given schemas',
+            'runcmd.3: {\'a\': \'n\'} is not valid under any of the given'
+            ' schema'
+        ]
+        logs = self.logs.getvalue()
+        for warning in expected_warnings:
+            self.assertIn(warning, logs)
+        self.assertIn('Failed to shellify', logs)
+
+    def test_handler_write_valid_runcmd_schema_to_file(self):
+        """Valid runcmd schema is written to a runcmd shell script."""
+        valid_config = {'runcmd': [['ls', '/']]}
+        cc = self._get_cloud('ubuntu')
+        cc_runcmd.handle('cc_runcmd', valid_config, cc, LOG, [])
+        runcmd_file = os.path.join(
+            self.new_root,
+            'var/lib/cloud/instances/iid-datasource-none/scripts/runcmd')
+        self.assertEqual("#!/bin/sh\n'ls' '/'\n", util.load_file(runcmd_file))
+        file_stat = os.stat(runcmd_file)
+        self.assertEqual(0o700, stat.S_IMODE(file_stat.st_mode))
+
+
+# vi: ts=4 expandtab