Add module 'write-files-deferred' executed in stage 'final' (#916)

The main idea is to introduce a second module that takes care of
writing files, but in the 'final' stage.

While the introduction of a second module would allow for choosing
the appropriate place withing the order of modules (and stages),
there is no addition top-level directive being added to the cloud
configuration schema. Instead, 'write-files' schema is being extended
to include a 'defer' attribute used only by the 'write-deffered-files'
modules.

The new module 'write-deferred-files' reuses as much as
possible of the 'write-files' functionality.

4 files changed, 112 insertions, 0 deletions

diff --git a/tests/integration_tests/modules/test_write_files.py b/tests/integration_tests/modules/test_write_files.py
index 15832ae3..1d532fac 100644
--- a/tests/integration_tests/modules/test_write_files.py
+++ b/tests/integration_tests/modules/test_write_files.py
@@ -21,6 +21,9 @@ B64_CONTENT = base64.b64encode(ASCII_TEXT.encode("utf-8"))
 #
 USER_DATA = """\
 #cloud-config
+users:
+-   default
+-   name: myuser
 write_files:
 -   encoding: b64
     content: {}
@@ -41,6 +44,12 @@ write_files:
         H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA=
     path: /root/file_gzip
     permissions: '0755'
+-   path: '/home/testuser/my-file'
+    content: |
+      echo 'hello world!'
+    defer: true
+    owner: 'myuser'
+    permissions: '0644'
 """.format(B64_CONTENT.decode("ascii"))
 
 
@@ -64,3 +73,15 @@ class TestWriteFiles:
     def test_write_files(self, cmd, expected_out, class_client):
         out = class_client.execute(cmd)
         assert expected_out in out
+
+    def test_write_files_deferred(self, class_client):
+        """Test that write files deferred works as expected.
+
+        Users get created after write_files module runs, so ensure that
+        with `defer: true`, the file gets written with correct ownership.
+        """
+        out = class_client.read_from_file("/home/testuser/my-file")
+        assert "echo 'hello world!'" == out
+        assert class_client.execute(
+            'stat -c "%U %a" /home/testuser/my-file'
+        ) == 'myuser 644'
diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py
index 727681d3..0af92805 100644
--- a/tests/unittests/test_handler/test_handler_write_files.py
+++ b/tests/unittests/test_handler/test_handler_write_files.py
@@ -189,6 +189,19 @@ class TestWriteFiles(FilesystemMockingTestCase):
             len(gz_aliases + gz_b64_aliases + b64_aliases) * len(datum))
         self.assertEqual(len(expected), flen_expected)
 
+    def test_deferred(self):
+        self.patchUtils(self.tmp)
+        file_path = '/tmp/deferred.file'
+        config = {
+            'write_files': [
+                {'path': file_path, 'defer': True}
+            ]
+        }
+        cc = self.tmp_cloud('ubuntu')
+        handle('cc_write_file', config, cc, LOG, [])
+        with self.assertRaises(FileNotFoundError):
+            util.load_file(file_path)
+
 
 class TestDecodePerms(CiTestCase):
 
diff --git a/tests/unittests/test_handler/test_handler_write_files_deferred.py b/tests/unittests/test_handler/test_handler_write_files_deferred.py
new file mode 100644
index 00000000..57b6934a
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_write_files_deferred.py
@@ -0,0 +1,77 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import tempfile
+import shutil
+
+from cloudinit.config.cc_write_files_deferred import (handle)
+from .test_handler_write_files import (VALID_SCHEMA)
+from cloudinit import log as logging
+from cloudinit import util
+
+from cloudinit.tests.helpers import (
+    CiTestCase, FilesystemMockingTestCase, mock, skipUnlessJsonSchema)
+
+LOG = logging.getLogger(__name__)
+
+
+@skipUnlessJsonSchema()
+@mock.patch('cloudinit.config.cc_write_files_deferred.write_files')
+class TestWriteFilesDeferredSchema(CiTestCase):
+
+    with_logs = True
+
+    def test_schema_validation_warns_invalid_value(self,
+                                                   m_write_files_deferred):
+        """If 'defer' is defined, it must be of type 'bool'."""
+
+        valid_config = {
+            'write_files': [
+                {**VALID_SCHEMA.get('write_files')[0], 'defer': True}
+            ]
+        }
+
+        invalid_config = {
+            'write_files': [
+                {**VALID_SCHEMA.get('write_files')[0], 'defer': str('no')}
+            ]
+        }
+
+        cc = self.tmp_cloud('ubuntu')
+        handle('cc_write_files_deferred', valid_config, cc, LOG, [])
+        self.assertNotIn('Invalid config:', self.logs.getvalue())
+        handle('cc_write_files_deferred', invalid_config, cc, LOG, [])
+        self.assertIn('Invalid config:', self.logs.getvalue())
+        self.assertIn("defer: 'no' is not of type 'boolean'",
+                      self.logs.getvalue())
+
+
+class TestWriteFilesDeferred(FilesystemMockingTestCase):
+
+    with_logs = True
+
+    def setUp(self):
+        super(TestWriteFilesDeferred, self).setUp()
+        self.tmp = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.tmp)
+
+    def test_filtering_deferred_files(self):
+        self.patchUtils(self.tmp)
+        expected = "hello world\n"
+        config = {
+            'write_files': [
+                {
+                    'path': '/tmp/deferred.file',
+                    'defer': True,
+                    'content': expected
+                },
+                {'path': '/tmp/not_deferred.file'}
+            ]
+        }
+        cc = self.tmp_cloud('ubuntu')
+        handle('cc_write_files_deferred', config, cc, LOG, [])
+        self.assertEqual(util.load_file('/tmp/deferred.file'), expected)
+        with self.assertRaises(FileNotFoundError):
+            util.load_file('/tmp/not_deferred.file')
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index 6f37ceb7..59f58f7c 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -34,6 +34,7 @@ class GetSchemaTest(CiTestCase):
                 'cc_ubuntu_advantage',
                 'cc_ubuntu_drivers',
                 'cc_write_files',
+                'cc_write_files_deferred',
                 'cc_zypper_add_repo',
                 'cc_chef'
             ],