Use instance-data-sensitive.json in jinja templates (SC-117) (#917)

instance-data.json redacts sensitive data for non-root users. Since user
data is consumed as root, we should be consuming the non-redacted data
instead.

LP: #1931392

3 files changed, 58 insertions, 18 deletions

diff --git a/cloudinit/handlers/jinja_template.py b/cloudinit/handlers/jinja_template.py
index aadfbf86..5033abbb 100644
--- a/cloudinit/handlers/jinja_template.py
+++ b/cloudinit/handlers/jinja_template.py
@@ -12,7 +12,7 @@ except ImportError:
 
 from cloudinit import handlers
 from cloudinit import log as logging
-from cloudinit.sources import INSTANCE_JSON_FILE
+from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE
 from cloudinit.templater import render_string, MISSING_JINJA_PREFIX
 from cloudinit.util import b64d, load_file, load_json, json_dumps
 
@@ -36,7 +36,8 @@ class JinjaTemplatePartHandler(handlers.Handler):
     def handle_part(self, data, ctype, filename, payload, frequency, headers):
         if ctype in handlers.CONTENT_SIGNALS:
             return
-        jinja_json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE)
+        jinja_json_file = os.path.join(
+            self.paths.run_dir, INSTANCE_JSON_SENSITIVE_FILE)
         rendered_payload = render_jinja_payload_from_file(
             payload, filename, jinja_json_file)
         if not rendered_payload:
diff --git a/doc/rtd/topics/instancedata.rst b/doc/rtd/topics/instancedata.rst
index 1850982c..6c17139f 100644
--- a/doc/rtd/topics/instancedata.rst
+++ b/doc/rtd/topics/instancedata.rst
@@ -509,14 +509,19 @@ EC2 instance:
 Using instance-data
 ===================
 
-As of cloud-init v. 18.4, any variables present in
-``/run/cloud-init/instance-data.json`` can be used in:
+As of cloud-init v. 18.4, any instance-data can be used in:
 
 * User-data scripts
 * Cloud config data
 * Command line interface via **cloud-init query** or
   **cloud-init devel render**
 
+This means that any variable present in
+``/run/cloud-init/instance-data-sensitive.json`` can be used,
+unless a non-root user is using the command line interface.
+In the non-root user case,
+``/run/cloud-init/instance-data.json`` will be used instead.
+
 Many clouds allow users to provide user-data to an instance at
 the time the instance is launched. Cloud-init supports a number of
 :ref:`user_data_formats`.
@@ -559,9 +564,39 @@ Below are some examples of providing these types of user-data:
    {%- endif %}
    ...
 
+One way to easily explore what Jinja variables are available on your machine
+is to use the ``cloud-init query --format`` (-f) commandline option which will
+render any Jinja syntax you use. Warnings or exceptions will be raised on
+invalid instance-data keys, paths or invalid syntax.
+
+.. code-block:: shell-session
+
+  # List all instance-data keys and values as root user
+  % sudo cloud-init query --all
+  {...}
+
+  # Introspect nested keys on an object
+  % cloud-init query -f "{{ds.keys()}}"
+  dict_keys(['meta_data', '_doc'])
+
+  # Test your Jinja rendering syntax on the command-line directly
+
+  # Failure to reference valid top-level instance-data key
+  % cloud-init query -f "{{invalid.instance-data.key}}"
+  WARNING: Ignoring jinja template for query commandline: 'invalid' is undefined
+
+  # Failure to reference valid dot-delimited key path on a known top-level key
+  % cloud-init query -f "{{v1.not_here}}"
+  WARNING: Could not render jinja template variables in file 'query commandline': 'not_here'
+  CI_MISSING_JINJA_VAR/not_here
+
+  # Test expected value using valid instance-data key path
+  % cloud-init query -f "My AMI: {{ds.meta_data.ami_id}}"
+  My AMI: ami-0fecc35d3c8ba8d60
+
 .. note::
   Trying to reference jinja variables that don't exist in
-  instance-data.json will result in warnings in ``/var/log/cloud-init.log``
+  instance-data will result in warnings in ``/var/log/cloud-init.log``
   and the following string in your rendered user-data:
   ``CI_MISSING_JINJA_VAR/<your_varname>``.
 
diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py
index c5675249..30293e9e 100644
--- a/tests/unittests/test_builtin_handlers.py
+++ b/tests/unittests/test_builtin_handlers.py
@@ -27,6 +27,8 @@ from cloudinit.handlers.upstart_job import UpstartJobPartHandler
 
 from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE)
 
+INSTANCE_DATA_FILE = 'instance-data-sensitive.json'
+
 
 class TestUpstartJobPartHandler(FilesystemMockingTestCase):
 
@@ -145,8 +147,8 @@ class TestJinjaTemplatePartHandler(CiTestCase):
         script_handler = ShellScriptPartHandler(self.paths)
         self.assertEqual(2, script_handler.handler_version)
 
-        # Create required instance-data.json file
-        instance_json = os.path.join(self.run_dir, 'instance-data.json')
+        # Create required instance data json file
+        instance_json = os.path.join(self.run_dir, INSTANCE_DATA_FILE)
         instance_data = {'topkey': 'echo himom'}
         util.write_file(instance_json, util.json_dumps(instance_data))
         h = JinjaTemplatePartHandler(
@@ -168,7 +170,7 @@ class TestJinjaTemplatePartHandler(CiTestCase):
         self.assertEqual(3, cloudcfg_handler.handler_version)
 
         # Create required instance-data.json file
-        instance_json = os.path.join(self.run_dir, 'instance-data.json')
+        instance_json = os.path.join(self.run_dir, INSTANCE_DATA_FILE)
         instance_data = {'topkey': {'sub': 'runcmd: [echo hi]'}}
         util.write_file(instance_json, util.json_dumps(instance_data))
         h = JinjaTemplatePartHandler(
@@ -198,8 +200,9 @@ class TestJinjaTemplatePartHandler(CiTestCase):
         script_file = os.path.join(script_handler.script_dir, 'part01')
         self.assertEqual(
             'Cannot render jinja template vars. Instance data not yet present'
-            ' at {}/instance-data.json'.format(
-                self.run_dir), str(context_manager.exception))
+            ' at {}/{}'.format(self.run_dir, INSTANCE_DATA_FILE),
+            str(context_manager.exception)
+        )
         self.assertFalse(
             os.path.exists(script_file),
             'Unexpected file created %s' % script_file)
@@ -207,7 +210,8 @@ class TestJinjaTemplatePartHandler(CiTestCase):
     def test_jinja_template_handle_errors_on_unreadable_instance_data(self):
         """If instance-data is unreadable, raise an error from handle_part."""
         script_handler = ShellScriptPartHandler(self.paths)
-        instance_json = os.path.join(self.run_dir, 'instance-data.json')
+        instance_json = os.path.join(
+            self.run_dir, INSTANCE_DATA_FILE)
         util.write_file(instance_json, util.json_dumps({}))
         h = JinjaTemplatePartHandler(
             self.paths, sub_handlers=[script_handler])
@@ -221,8 +225,8 @@ class TestJinjaTemplatePartHandler(CiTestCase):
                     frequency='freq', headers='headers')
         script_file = os.path.join(script_handler.script_dir, 'part01')
         self.assertEqual(
-            'Cannot render jinja template vars. No read permission on'
-            " '{rdir}/instance-data.json'. Try sudo".format(rdir=self.run_dir),
+            "Cannot render jinja template vars. No read permission on "
+            "'{}/{}'. Try sudo".format(self.run_dir, INSTANCE_DATA_FILE),
             str(context_manager.exception))
         self.assertFalse(
             os.path.exists(script_file),
@@ -230,9 +234,9 @@ class TestJinjaTemplatePartHandler(CiTestCase):
 
     @skipUnlessJinja()
     def test_jinja_template_handle_renders_jinja_content(self):
-        """When present, render jinja variables from instance-data.json."""
+        """When present, render jinja variables from instance data"""
         script_handler = ShellScriptPartHandler(self.paths)
-        instance_json = os.path.join(self.run_dir, 'instance-data.json')
+        instance_json = os.path.join(self.run_dir, INSTANCE_DATA_FILE)
         instance_data = {'topkey': {'subkey': 'echo himom'}}
         util.write_file(instance_json, util.json_dumps(instance_data))
         h = JinjaTemplatePartHandler(
@@ -247,8 +251,8 @@ class TestJinjaTemplatePartHandler(CiTestCase):
             frequency='freq', headers='headers')
         script_file = os.path.join(script_handler.script_dir, 'part01')
         self.assertNotIn(
-            'Instance data not yet present at {}/instance-data.json'.format(
-                self.run_dir),
+            'Instance data not yet present at {}/{}'.format(
+                self.run_dir, INSTANCE_DATA_FILE),
             self.logs.getvalue())
         self.assertEqual(
             '#!/bin/bash\necho himom', util.load_file(script_file))
@@ -257,7 +261,7 @@ class TestJinjaTemplatePartHandler(CiTestCase):
     def test_jinja_template_handle_renders_jinja_content_missing_keys(self):
         """When specified jinja variable is undefined, log a warning."""
         script_handler = ShellScriptPartHandler(self.paths)
-        instance_json = os.path.join(self.run_dir, 'instance-data.json')
+        instance_json = os.path.join(self.run_dir, INSTANCE_DATA_FILE)
         instance_data = {'topkey': {'subkey': 'echo himom'}}
         util.write_file(instance_json, util.json_dumps(instance_data))
         h = JinjaTemplatePartHandler(