schema: add json schema for write_files module (#152)

Add schema definition to cc_write_files.py

Cloud-config containing write_files config directives will now
emit warnings for invalid config keys or values for the write_files
module.

Add an extension to JSON schema's draft4validator to permit either
binary or text values for 'string' objects.

This allows for JSON schema validating the YAML declaration of binary
valiues in cloud-config using YAML's  '!!binary' syntax.

Add the ability to pass a specific module name to
`cloud-init devel schema --docs <module_name>|all` to optionally
limit doc output during development to a single schema doc.

3 files changed, 89 insertions, 7 deletions

diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index e57c15d1..43d996b9 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -214,14 +214,14 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
         self.assertEqual(1, exit_code)
         # Known whitebox output from schema subcommand
         self.assertEqual(
-            'Expected either --config-file argument or --doc\n',
+            'Expected either --config-file argument or --docs\n',
             self.stderr.getvalue())
 
     def test_wb_devel_schema_subcommand_doc_content(self):
         """Validate that doc content is sane from known examples."""
         stdout = io.StringIO()
         self.patchStdoutAndStderr(stdout=stdout)
-        self._call_main(['cloud-init', 'devel', 'schema', '--doc'])
+        self._call_main(['cloud-init', 'devel', 'schema', '--docs', 'all'])
         expected_doc_sections = [
             '**Supported distros:** all',
             '**Supported distros:** centos, debian, fedora',
diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py
index ed0a4da2..727681d3 100644
--- a/tests/unittests/test_handler/test_handler_write_files.py
+++ b/tests/unittests/test_handler/test_handler_write_files.py
@@ -1,15 +1,19 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import base64
+import copy
 import gzip
 import io
 import shutil
 import tempfile
 
+from cloudinit.config.cc_write_files import (
+    handle, decode_perms, write_files)
 from cloudinit import log as logging
 from cloudinit import util
-from cloudinit.config.cc_write_files import write_files, decode_perms
-from cloudinit.tests.helpers import CiTestCase, FilesystemMockingTestCase
+
+from cloudinit.tests.helpers import (
+    CiTestCase, FilesystemMockingTestCase, mock, skipUnlessJsonSchema)
 
 LOG = logging.getLogger(__name__)
 
@@ -36,13 +40,90 @@ YAML_CONTENT_EXPECTED = {
     '/tmp/message': "hi mom line 1\nhi mom line 2\n",
 }
 
+VALID_SCHEMA = {
+    'write_files': [
+        {'append': False, 'content': 'a', 'encoding': 'gzip', 'owner': 'jeff',
+         'path': '/some', 'permissions': '0777'}
+    ]
+}
+
+INVALID_SCHEMA = {  # Dropped required path key
+    'write_files': [
+        {'append': False, 'content': 'a', 'encoding': 'gzip', 'owner': 'jeff',
+         'permissions': '0777'}
+    ]
+}
+
+
+@skipUnlessJsonSchema()
+@mock.patch('cloudinit.config.cc_write_files.write_files')
+class TestWriteFilesSchema(CiTestCase):
+
+    with_logs = True
+
+    def test_schema_validation_warns_missing_path(self, m_write_files):
+        """The only required file item property is 'path'."""
+        cc = self.tmp_cloud('ubuntu')
+        valid_config = {'write_files': [{'path': '/some/path'}]}
+        handle('cc_write_file', valid_config, cc, LOG, [])
+        self.assertNotIn('Invalid config:', self.logs.getvalue())
+        handle('cc_write_file', INVALID_SCHEMA, cc, LOG, [])
+        self.assertIn('Invalid config:', self.logs.getvalue())
+        self.assertIn("'path' is a required property", self.logs.getvalue())
+
+    def test_schema_validation_warns_non_string_type_for_files(
+            self, m_write_files):
+        """Schema validation warns of non-string values for each file item."""
+        cc = self.tmp_cloud('ubuntu')
+        for key in VALID_SCHEMA['write_files'][0].keys():
+            if key == 'append':
+                key_type = 'boolean'
+            else:
+                key_type = 'string'
+            invalid_config = copy.deepcopy(VALID_SCHEMA)
+            invalid_config['write_files'][0][key] = 1
+            handle('cc_write_file', invalid_config, cc, LOG, [])
+            self.assertIn(
+                mock.call('cc_write_file', invalid_config['write_files']),
+                m_write_files.call_args_list)
+            self.assertIn(
+                'write_files.0.%s: 1 is not of type \'%s\'' % (key, key_type),
+                self.logs.getvalue())
+        self.assertIn('Invalid config:', self.logs.getvalue())
+
+    def test_schema_validation_warns_on_additional_undefined_propertes(
+            self, m_write_files):
+        """Schema validation warns on additional undefined file properties."""
+        cc = self.tmp_cloud('ubuntu')
+        invalid_config = copy.deepcopy(VALID_SCHEMA)
+        invalid_config['write_files'][0]['bogus'] = 'value'
+        handle('cc_write_file', invalid_config, cc, LOG, [])
+        self.assertIn(
+            "Invalid config:\nwrite_files.0: Additional properties"
+            " are not allowed ('bogus' was unexpected)",
+            self.logs.getvalue())
+
 
 class TestWriteFiles(FilesystemMockingTestCase):
+
+    with_logs = True
+
     def setUp(self):
         super(TestWriteFiles, self).setUp()
         self.tmp = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, self.tmp)
 
+    @skipUnlessJsonSchema()
+    def test_handler_schema_validation_warns_non_array_type(self):
+        """Schema validation warns of non-array value."""
+        invalid_config = {'write_files': 1}
+        cc = self.tmp_cloud('ubuntu')
+        with self.assertRaises(TypeError):
+            handle('cc_write_file', invalid_config, cc, LOG, [])
+        self.assertIn(
+            'Invalid config:\nwrite_files: 1 is not of type \'array\'',
+            self.logs.getvalue())
+
     def test_simple(self):
         self.patchUtils(self.tmp)
         expected = "hello world\n"
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index 98628120..2d4536d9 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -29,6 +29,7 @@ class GetSchemaTest(CiTestCase):
                 'cc_snap',
                 'cc_ubuntu_advantage',
                 'cc_ubuntu_drivers',
+                'cc_write_files',
                 'cc_zypper_add_repo'
             ],
             [subschema['id'] for subschema in schema['allOf']])
@@ -351,7 +352,7 @@ class MainTest(CiTestCase):
                     main()
         self.assertEqual(1, context_manager.exception.code)
         self.assertEqual(
-            'Expected either --config-file argument or --doc\n',
+            'Expected either --config-file argument or --docs\n',
             m_stderr.getvalue())
 
     def test_main_absent_config_file(self):
@@ -367,8 +368,8 @@ class MainTest(CiTestCase):
             m_stderr.getvalue())
 
     def test_main_prints_docs(self):
-        """When --doc parameter is provided, main generates documentation."""
-        myargs = ['mycmd', '--doc']
+        """When --docs parameter is provided, main generates documentation."""
+        myargs = ['mycmd', '--docs', 'all']
         with mock.patch('sys.argv', myargs):
             with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
                 self.assertEqual(0, main(), 'Expected 0 exit code')