From 05b0e35026db3789c56ee9f8192d4a81067325e5 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Thu, 10 Jun 2021 14:24:51 -0500 Subject: Use instance-data-sensitive.json in jinja templates (SC-117) (#917) instance-data.json redacts sensitive data for non-root users. Since user data is consumed as root, we should be consuming the non-redacted data instead. LP: #1931392 --- cloudinit/handlers/jinja_template.py | 5 ++-- doc/rtd/topics/instancedata.rst | 41 +++++++++++++++++++++++++++++--- tests/unittests/test_builtin_handlers.py | 30 +++++++++++++---------- 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/cloudinit/handlers/jinja_template.py b/cloudinit/handlers/jinja_template.py index aadfbf86..5033abbb 100644 --- a/cloudinit/handlers/jinja_template.py +++ b/cloudinit/handlers/jinja_template.py @@ -12,7 +12,7 @@ except ImportError: from cloudinit import handlers from cloudinit import log as logging -from cloudinit.sources import INSTANCE_JSON_FILE +from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE from cloudinit.templater import render_string, MISSING_JINJA_PREFIX from cloudinit.util import b64d, load_file, load_json, json_dumps @@ -36,7 +36,8 @@ class JinjaTemplatePartHandler(handlers.Handler): def handle_part(self, data, ctype, filename, payload, frequency, headers): if ctype in handlers.CONTENT_SIGNALS: return - jinja_json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE) + jinja_json_file = os.path.join( + self.paths.run_dir, INSTANCE_JSON_SENSITIVE_FILE) rendered_payload = render_jinja_payload_from_file( payload, filename, jinja_json_file) if not rendered_payload: diff --git a/doc/rtd/topics/instancedata.rst b/doc/rtd/topics/instancedata.rst index 1850982c..6c17139f 100644 --- a/doc/rtd/topics/instancedata.rst +++ b/doc/rtd/topics/instancedata.rst @@ -509,14 +509,19 @@ EC2 instance: Using instance-data =================== -As of cloud-init v. 18.4, any variables present in -``/run/cloud-init/instance-data.json`` can be used in: +As of cloud-init v. 18.4, any instance-data can be used in: * User-data scripts * Cloud config data * Command line interface via **cloud-init query** or **cloud-init devel render** +This means that any variable present in +``/run/cloud-init/instance-data-sensitive.json`` can be used, +unless a non-root user is using the command line interface. +In the non-root user case, +``/run/cloud-init/instance-data.json`` will be used instead. + Many clouds allow users to provide user-data to an instance at the time the instance is launched. Cloud-init supports a number of :ref:`user_data_formats`. @@ -559,9 +564,39 @@ Below are some examples of providing these types of user-data: {%- endif %} ... +One way to easily explore what Jinja variables are available on your machine +is to use the ``cloud-init query --format`` (-f) commandline option which will +render any Jinja syntax you use. Warnings or exceptions will be raised on +invalid instance-data keys, paths or invalid syntax. + +.. code-block:: shell-session + + # List all instance-data keys and values as root user + % sudo cloud-init query --all + {...} + + # Introspect nested keys on an object + % cloud-init query -f "{{ds.keys()}}" + dict_keys(['meta_data', '_doc']) + + # Test your Jinja rendering syntax on the command-line directly + + # Failure to reference valid top-level instance-data key + % cloud-init query -f "{{invalid.instance-data.key}}" + WARNING: Ignoring jinja template for query commandline: 'invalid' is undefined + + # Failure to reference valid dot-delimited key path on a known top-level key + % cloud-init query -f "{{v1.not_here}}" + WARNING: Could not render jinja template variables in file 'query commandline': 'not_here' + CI_MISSING_JINJA_VAR/not_here + + # Test expected value using valid instance-data key path + % cloud-init query -f "My AMI: {{ds.meta_data.ami_id}}" + My AMI: ami-0fecc35d3c8ba8d60 + .. note:: Trying to reference jinja variables that don't exist in - instance-data.json will result in warnings in ``/var/log/cloud-init.log`` + instance-data will result in warnings in ``/var/log/cloud-init.log`` and the following string in your rendered user-data: ``CI_MISSING_JINJA_VAR/``. diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index c5675249..30293e9e 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -27,6 +27,8 @@ from cloudinit.handlers.upstart_job import UpstartJobPartHandler from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) +INSTANCE_DATA_FILE = 'instance-data-sensitive.json' + class TestUpstartJobPartHandler(FilesystemMockingTestCase): @@ -145,8 +147,8 @@ class TestJinjaTemplatePartHandler(CiTestCase): script_handler = ShellScriptPartHandler(self.paths) self.assertEqual(2, script_handler.handler_version) - # Create required instance-data.json file - instance_json = os.path.join(self.run_dir, 'instance-data.json') + # Create required instance data json file + instance_json = os.path.join(self.run_dir, INSTANCE_DATA_FILE) instance_data = {'topkey': 'echo himom'} util.write_file(instance_json, util.json_dumps(instance_data)) h = JinjaTemplatePartHandler( @@ -168,7 +170,7 @@ class TestJinjaTemplatePartHandler(CiTestCase): self.assertEqual(3, cloudcfg_handler.handler_version) # Create required instance-data.json file - instance_json = os.path.join(self.run_dir, 'instance-data.json') + instance_json = os.path.join(self.run_dir, INSTANCE_DATA_FILE) instance_data = {'topkey': {'sub': 'runcmd: [echo hi]'}} util.write_file(instance_json, util.json_dumps(instance_data)) h = JinjaTemplatePartHandler( @@ -198,8 +200,9 @@ class TestJinjaTemplatePartHandler(CiTestCase): script_file = os.path.join(script_handler.script_dir, 'part01') self.assertEqual( 'Cannot render jinja template vars. Instance data not yet present' - ' at {}/instance-data.json'.format( - self.run_dir), str(context_manager.exception)) + ' at {}/{}'.format(self.run_dir, INSTANCE_DATA_FILE), + str(context_manager.exception) + ) self.assertFalse( os.path.exists(script_file), 'Unexpected file created %s' % script_file) @@ -207,7 +210,8 @@ class TestJinjaTemplatePartHandler(CiTestCase): def test_jinja_template_handle_errors_on_unreadable_instance_data(self): """If instance-data is unreadable, raise an error from handle_part.""" script_handler = ShellScriptPartHandler(self.paths) - instance_json = os.path.join(self.run_dir, 'instance-data.json') + instance_json = os.path.join( + self.run_dir, INSTANCE_DATA_FILE) util.write_file(instance_json, util.json_dumps({})) h = JinjaTemplatePartHandler( self.paths, sub_handlers=[script_handler]) @@ -221,8 +225,8 @@ class TestJinjaTemplatePartHandler(CiTestCase): frequency='freq', headers='headers') script_file = os.path.join(script_handler.script_dir, 'part01') self.assertEqual( - 'Cannot render jinja template vars. No read permission on' - " '{rdir}/instance-data.json'. Try sudo".format(rdir=self.run_dir), + "Cannot render jinja template vars. No read permission on " + "'{}/{}'. Try sudo".format(self.run_dir, INSTANCE_DATA_FILE), str(context_manager.exception)) self.assertFalse( os.path.exists(script_file), @@ -230,9 +234,9 @@ class TestJinjaTemplatePartHandler(CiTestCase): @skipUnlessJinja() def test_jinja_template_handle_renders_jinja_content(self): - """When present, render jinja variables from instance-data.json.""" + """When present, render jinja variables from instance data""" script_handler = ShellScriptPartHandler(self.paths) - instance_json = os.path.join(self.run_dir, 'instance-data.json') + instance_json = os.path.join(self.run_dir, INSTANCE_DATA_FILE) instance_data = {'topkey': {'subkey': 'echo himom'}} util.write_file(instance_json, util.json_dumps(instance_data)) h = JinjaTemplatePartHandler( @@ -247,8 +251,8 @@ class TestJinjaTemplatePartHandler(CiTestCase): frequency='freq', headers='headers') script_file = os.path.join(script_handler.script_dir, 'part01') self.assertNotIn( - 'Instance data not yet present at {}/instance-data.json'.format( - self.run_dir), + 'Instance data not yet present at {}/{}'.format( + self.run_dir, INSTANCE_DATA_FILE), self.logs.getvalue()) self.assertEqual( '#!/bin/bash\necho himom', util.load_file(script_file)) @@ -257,7 +261,7 @@ class TestJinjaTemplatePartHandler(CiTestCase): def test_jinja_template_handle_renders_jinja_content_missing_keys(self): """When specified jinja variable is undefined, log a warning.""" script_handler = ShellScriptPartHandler(self.paths) - instance_json = os.path.join(self.run_dir, 'instance-data.json') + instance_json = os.path.join(self.run_dir, INSTANCE_DATA_FILE) instance_data = {'topkey': {'subkey': 'echo himom'}} util.write_file(instance_json, util.json_dumps(instance_data)) h = JinjaTemplatePartHandler( -- cgit v1.2.3