Add module 'write-files-deferred' executed in stage 'final' (#916)

The main idea is to introduce a second module that takes care of
writing files, but in the 'final' stage.

While the introduction of a second module would allow for choosing
the appropriate place withing the order of modules (and stages),
there is no addition top-level directive being added to the cloud
configuration schema. Instead, 'write-files' schema is being extended
to include a 'defer' attribute used only by the 'write-deffered-files'
modules.

The new module 'write-deferred-files' reuses as much as
possible of the 'write-files' functionality.

8 files changed, 206 insertions, 4 deletions

diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py
index 8601e707..41c75fa2 100644
--- a/cloudinit/config/cc_write_files.py
+++ b/cloudinit/config/cc_write_files.py
@@ -21,6 +21,7 @@ frequency = PER_INSTANCE
 
 DEFAULT_OWNER = "root:root"
 DEFAULT_PERMS = 0o644
+DEFAULT_DEFER = False
 UNKNOWN_ENC = 'text/plain'
 
 LOG = logging.getLogger(__name__)
@@ -90,6 +91,24 @@ schema = {
         # Create an empty file on the system
         write_files:
         - path: /root/CLOUD_INIT_WAS_HERE
+        """),
+        dedent("""\
+        # Defer writing the file until after the package (Nginx) is
+        # installed and its user is created alongside
+        write_files:
+        - path: /etc/nginx/conf.d/example.com.conf
+          content: |
+            server {
+                server_name example.com;
+                listen 80;
+                root /var/www;
+                location / {
+                    try_files $uri $uri/ $uri.html =404;
+                }
+            }
+          owner: 'nginx:nginx'
+          permissions: '0640'
+          defer: true
         """)],
     'frequency': frequency,
     'type': 'object',
@@ -151,6 +170,15 @@ schema = {
                             ``path`` exists. Default: **false**.
                         """),
                     },
+                    'defer': {
+                        'type': 'boolean',
+                        'default': DEFAULT_DEFER,
+                        'description': dedent("""\
+                            Defer writing the file until 'final' stage, after
+                            users were created, and packages were installed.
+                            Default: **{defer}**.
+                        """.format(defer=DEFAULT_DEFER)),
+                    },
                 },
                 'required': ['path'],
                 'additionalProperties': False
@@ -163,13 +191,18 @@ __doc__ = get_schema_doc(schema)  # Supplement python help()
 
 
 def handle(name, cfg, _cloud, log, _args):
-    files = cfg.get('write_files')
-    if not files:
+    validate_cloudconfig_schema(cfg, schema)
+    file_list = cfg.get('write_files', [])
+    filtered_files = [
+        f for f in file_list if not util.get_cfg_option_bool(f,
+                                                             'defer',
+                                                             DEFAULT_DEFER)
+    ]
+    if not filtered_files:
         log.debug(("Skipping module named %s,"
                    " no/empty 'write_files' key in configuration"), name)
         return
-    validate_cloudconfig_schema(cfg, schema)
-    write_files(name, files)
+    write_files(name, filtered_files)
 
 
 def canonicalize_extraction(encoding_type):
diff --git a/cloudinit/config/cc_write_files_deferred.py b/cloudinit/config/cc_write_files_deferred.py
new file mode 100644
index 00000000..0c75aa22
--- /dev/null
+++ b/cloudinit/config/cc_write_files_deferred.py
@@ -0,0 +1,55 @@
+# Copyright (C) 2021 Canonical Ltd.
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Defer writing certain files"""
+
+from textwrap import dedent
+
+from cloudinit.config.schema import validate_cloudconfig_schema
+from cloudinit import util
+from cloudinit.config.cc_write_files import (
+    schema as write_files_schema, write_files, DEFAULT_DEFER)
+
+
+schema = util.mergemanydict([
+    {
+        'id': 'cc_write_files_deferred',
+        'name': 'Write Deferred Files',
+        'title': dedent("""\
+            write certain files, whose creation as been deferred, during
+            final stage
+        """),
+        'description': dedent("""\
+            This module is based on `'Write Files' <write-files>`__, and
+            will handle all files from the write_files list, that have been
+            marked as deferred and thus are not being processed by the
+            write-files module.
+
+            *Please note that his module is not exposed to the user through
+            its own dedicated top-level directive.*
+        """)
+    },
+    write_files_schema
+])
+
+# Not exposed, because related modules should document this behaviour
+__doc__ = None
+
+
+def handle(name, cfg, _cloud, log, _args):
+    validate_cloudconfig_schema(cfg, schema)
+    file_list = cfg.get('write_files', [])
+    filtered_files = [
+        f for f in file_list if util.get_cfg_option_bool(f,
+                                                         'defer',
+                                                         DEFAULT_DEFER)
+    ]
+    if not filtered_files:
+        log.debug(("Skipping module named %s,"
+                   " no deferred file defined in configuration"), name)
+        return
+    write_files(name, filtered_files)
+
+
+# vi: ts=4 expandtab
diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl
index de1d75e5..66c48fd5 100644
--- a/config/cloud.cfg.tmpl
+++ b/config/cloud.cfg.tmpl
@@ -151,6 +151,7 @@ cloud_final_modules:
 {% if variant in ["ubuntu", "unknown"] %}
  - ubuntu-drivers
 {% endif %}
+ - write-files-deferred
  - puppet
  - chef
  - mcollective
diff --git a/tests/integration_tests/modules/test_write_files.py b/tests/integration_tests/modules/test_write_files.py
index 15832ae3..1d532fac 100644
--- a/tests/integration_tests/modules/test_write_files.py
+++ b/tests/integration_tests/modules/test_write_files.py
@@ -21,6 +21,9 @@ B64_CONTENT = base64.b64encode(ASCII_TEXT.encode("utf-8"))
 #
 USER_DATA = """\
 #cloud-config
+users:
+-   default
+-   name: myuser
 write_files:
 -   encoding: b64
     content: {}
@@ -41,6 +44,12 @@ write_files:
         H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA=
     path: /root/file_gzip
     permissions: '0755'
+-   path: '/home/testuser/my-file'
+    content: |
+      echo 'hello world!'
+    defer: true
+    owner: 'myuser'
+    permissions: '0644'
 """.format(B64_CONTENT.decode("ascii"))
 
 
@@ -64,3 +73,15 @@ class TestWriteFiles:
     def test_write_files(self, cmd, expected_out, class_client):
         out = class_client.execute(cmd)
         assert expected_out in out
+
+    def test_write_files_deferred(self, class_client):
+        """Test that write files deferred works as expected.
+
+        Users get created after write_files module runs, so ensure that
+        with `defer: true`, the file gets written with correct ownership.
+        """
+        out = class_client.read_from_file("/home/testuser/my-file")
+        assert "echo 'hello world!'" == out
+        assert class_client.execute(
+            'stat -c "%U %a" /home/testuser/my-file'
+        ) == 'myuser 644'
diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py
index 727681d3..0af92805 100644
--- a/tests/unittests/test_handler/test_handler_write_files.py
+++ b/tests/unittests/test_handler/test_handler_write_files.py
@@ -189,6 +189,19 @@ class TestWriteFiles(FilesystemMockingTestCase):
             len(gz_aliases + gz_b64_aliases + b64_aliases) * len(datum))
         self.assertEqual(len(expected), flen_expected)
 
+    def test_deferred(self):
+        self.patchUtils(self.tmp)
+        file_path = '/tmp/deferred.file'
+        config = {
+            'write_files': [
+                {'path': file_path, 'defer': True}
+            ]
+        }
+        cc = self.tmp_cloud('ubuntu')
+        handle('cc_write_file', config, cc, LOG, [])
+        with self.assertRaises(FileNotFoundError):
+            util.load_file(file_path)
+
 
 class TestDecodePerms(CiTestCase):
 
diff --git a/tests/unittests/test_handler/test_handler_write_files_deferred.py b/tests/unittests/test_handler/test_handler_write_files_deferred.py
new file mode 100644
index 00000000..57b6934a
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_write_files_deferred.py
@@ -0,0 +1,77 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import tempfile
+import shutil
+
+from cloudinit.config.cc_write_files_deferred import (handle)
+from .test_handler_write_files import (VALID_SCHEMA)
+from cloudinit import log as logging
+from cloudinit import util
+
+from cloudinit.tests.helpers import (
+    CiTestCase, FilesystemMockingTestCase, mock, skipUnlessJsonSchema)
+
+LOG = logging.getLogger(__name__)
+
+
+@skipUnlessJsonSchema()
+@mock.patch('cloudinit.config.cc_write_files_deferred.write_files')
+class TestWriteFilesDeferredSchema(CiTestCase):
+
+    with_logs = True
+
+    def test_schema_validation_warns_invalid_value(self,
+                                                   m_write_files_deferred):
+        """If 'defer' is defined, it must be of type 'bool'."""
+
+        valid_config = {
+            'write_files': [
+                {**VALID_SCHEMA.get('write_files')[0], 'defer': True}
+            ]
+        }
+
+        invalid_config = {
+            'write_files': [
+                {**VALID_SCHEMA.get('write_files')[0], 'defer': str('no')}
+            ]
+        }
+
+        cc = self.tmp_cloud('ubuntu')
+        handle('cc_write_files_deferred', valid_config, cc, LOG, [])
+        self.assertNotIn('Invalid config:', self.logs.getvalue())
+        handle('cc_write_files_deferred', invalid_config, cc, LOG, [])
+        self.assertIn('Invalid config:', self.logs.getvalue())
+        self.assertIn("defer: 'no' is not of type 'boolean'",
+                      self.logs.getvalue())
+
+
+class TestWriteFilesDeferred(FilesystemMockingTestCase):
+
+    with_logs = True
+
+    def setUp(self):
+        super(TestWriteFilesDeferred, self).setUp()
+        self.tmp = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.tmp)
+
+    def test_filtering_deferred_files(self):
+        self.patchUtils(self.tmp)
+        expected = "hello world\n"
+        config = {
+            'write_files': [
+                {
+                    'path': '/tmp/deferred.file',
+                    'defer': True,
+                    'content': expected
+                },
+                {'path': '/tmp/not_deferred.file'}
+            ]
+        }
+        cc = self.tmp_cloud('ubuntu')
+        handle('cc_write_files_deferred', config, cc, LOG, [])
+        self.assertEqual(util.load_file('/tmp/deferred.file'), expected)
+        with self.assertRaises(FileNotFoundError):
+            util.load_file('/tmp/not_deferred.file')
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index 6f37ceb7..59f58f7c 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -34,6 +34,7 @@ class GetSchemaTest(CiTestCase):
                 'cc_ubuntu_advantage',
                 'cc_ubuntu_drivers',
                 'cc_write_files',
+                'cc_write_files_deferred',
                 'cc_zypper_add_repo',
                 'cc_chef'
             ],
diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers
index 99f7d99c..fac3fcec 100644
--- a/tools/.github-cla-signers
+++ b/tools/.github-cla-signers
@@ -42,6 +42,7 @@ jshen28
 klausenbusk
 landon912
 lucasmoura
+lucendio
 lungj
 mal
 mamercad