diff options
| -rw-r--r-- | cloudinit/cmd/query.py | 134 | ||||
| -rw-r--r-- | cloudinit/cmd/tests/test_query.py | 71 | ||||
| -rw-r--r-- | cloudinit/handlers/jinja_template.py | 48 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceLXD.py | 26 | ||||
| -rw-r--r-- | cloudinit/sources/tests/test_lxd.py | 71 | ||||
| -rw-r--r-- | doc/rtd/topics/instancedata.rst | 16 | ||||
| -rw-r--r-- | tests/integration_tests/datasources/test_lxd_discovery.py | 17 | ||||
| -rw-r--r-- | tests/integration_tests/modules/test_jinja_templating.py | 12 | ||||
| -rw-r--r-- | tests/unittests/test_builtin_handlers.py | 68 |
9 files changed, 326 insertions, 137 deletions
diff --git a/cloudinit/cmd/query.py b/cloudinit/cmd/query.py index 07db9552..e53cd855 100644 --- a/cloudinit/cmd/query.py +++ b/cloudinit/cmd/query.py @@ -19,7 +19,10 @@ import os import sys from cloudinit.handlers.jinja_template import ( - convert_jinja_instance_data, render_jinja_payload) + convert_jinja_instance_data, + get_jinja_variable_alias, + render_jinja_payload +) from cloudinit.cmd.devel import addLogHandlerCLI, read_cfg_paths from cloudinit import log from cloudinit.sources import ( @@ -93,22 +96,24 @@ def load_userdata(ud_file_path): return util.decomp_gzip(bdata, quiet=False, decode=True) -def handle_args(name, args): - """Handle calls to 'cloud-init query' as a subcommand.""" - paths = None - addLogHandlerCLI(LOG, log.DEBUG if args.debug else log.WARNING) - if not any([args.list_keys, args.varname, args.format, args.dump_all]): - LOG.error( - 'Expected one of the options: --all, --format,' - ' --list-keys or varname') - get_parser().print_help() - return 1 +def _read_instance_data(instance_data, user_data, vendor_data) -> dict: + """Return a dict of merged instance-data, vendordata and userdata. + The dict will contain supplemental userdata and vendordata keys sourced + from default user-data and vendor-data files. + + Non-root users will have redacted INSTANCE_JSON_FILE content and redacted + vendordata and userdata values. + + :raise: IOError/OSError on absence of instance-data.json file or invalid + access perms. + """ + paths = None uid = os.getuid() - if not all([args.instance_data, args.user_data, args.vendor_data]): + if not all([instance_data, user_data, vendor_data]): paths = read_cfg_paths() - if args.instance_data: - instance_data_fn = args.instance_data + if instance_data: + instance_data_fn = instance_data else: redacted_data_fn = os.path.join(paths.run_dir, INSTANCE_JSON_FILE) if uid == 0: @@ -124,12 +129,12 @@ def handle_args(name, args): instance_data_fn = redacted_data_fn else: instance_data_fn = redacted_data_fn - if args.user_data: - user_data_fn = args.user_data + if user_data: + user_data_fn = user_data else: user_data_fn = os.path.join(paths.instance_link, 'user-data.txt') - if args.vendor_data: - vendor_data_fn = args.vendor_data + if vendor_data: + vendor_data_fn = vendor_data else: vendor_data_fn = os.path.join(paths.instance_link, 'vendor-data.txt') @@ -140,7 +145,7 @@ def handle_args(name, args): LOG.error("No read permission on '%s'. Try sudo", instance_data_fn) else: LOG.error('Missing instance-data file: %s', instance_data_fn) - return 1 + raise instance_data = util.load_json(instance_json) if uid != 0: @@ -151,6 +156,65 @@ def handle_args(name, args): else: instance_data['userdata'] = load_userdata(user_data_fn) instance_data['vendordata'] = load_userdata(vendor_data_fn) + return instance_data + + +def _find_instance_data_leaf_by_varname_path( + jinja_vars_without_aliases: dict, jinja_vars_with_aliases: dict, + varname: str, list_keys: bool +): + """Return the value of the dot-delimited varname path in instance-data + + Split a dot-delimited jinja variable name path into components, walk the + path components into the instance_data and look up a matching jinja + variable name or cloud-init's underscore-delimited key aliases. + + :raises: ValueError when varname represents an invalid key name or path or + if list-keys is provided by varname isn't a dict object. + """ + walked_key_path = "" + response = jinja_vars_without_aliases + for key_path_part in varname.split('.'): + try: + # Walk key path using complete aliases dict, yet response + # should only contain jinja_without_aliases + jinja_vars_with_aliases = jinja_vars_with_aliases[key_path_part] + except KeyError as e: + if walked_key_path: + msg = "instance-data '{key_path}' has no '{leaf}'".format( + leaf=key_path_part, key_path=walked_key_path + ) + else: + msg = "Undefined instance-data key '{}'".format(varname) + raise ValueError(msg) from e + if key_path_part in response: + response = response[key_path_part] + else: # We are an underscore_delimited key alias + for key in response: + if get_jinja_variable_alias(key) == key_path_part: + response = response[key] + break + if walked_key_path: + walked_key_path += "." + walked_key_path += key_path_part + return response + + +def handle_args(name, args): + """Handle calls to 'cloud-init query' as a subcommand.""" + addLogHandlerCLI(LOG, log.DEBUG if args.debug else log.WARNING) + if not any([args.list_keys, args.varname, args.format, args.dump_all]): + LOG.error( + 'Expected one of the options: --all, --format,' + ' --list-keys or varname') + get_parser().print_help() + return 1 + try: + instance_data = _read_instance_data( + args.instance_data, args.user_data, args.vendor_data + ) + except (IOError, OSError): + return 1 if args.format: payload = '## template: jinja\n{fmt}'.format(fmt=args.format) rendered_payload = render_jinja_payload( @@ -162,20 +226,32 @@ def handle_args(name, args): return 0 return 1 + # If not rendering a structured format above, query output will be either: + # - JSON dump of all instance-data/jinja variables + # - JSON dump of a value at an dict path into the instance-data dict. + # - a list of keys for a specific dict path into the instance-data dict. response = convert_jinja_instance_data(instance_data) if args.varname: + jinja_vars_with_aliases = convert_jinja_instance_data( + instance_data, include_key_aliases=True + ) try: - for var in args.varname.split('.'): - response = response[var] - except KeyError: - LOG.error('Undefined instance-data key %s', args.varname) + response = _find_instance_data_leaf_by_varname_path( + jinja_vars_without_aliases=response, + jinja_vars_with_aliases=jinja_vars_with_aliases, + varname=args.varname, + list_keys=args.list_keys + ) + except (KeyError, ValueError) as e: + LOG.error(e) + return 1 + if args.list_keys: + if not isinstance(response, dict): + LOG.error( + "--list-keys provided but '%s' is not a dict", + args.varname + ) return 1 - if args.list_keys: - if not isinstance(response, dict): - LOG.error("--list-keys provided but '%s' is not a dict", var) - return 1 - response = '\n'.join(sorted(response.keys())) - elif args.list_keys: response = '\n'.join(sorted(response.keys())) if not isinstance(response, str): response = util.json_dumps(response) diff --git a/cloudinit/cmd/tests/test_query.py b/cloudinit/cmd/tests/test_query.py index c258d321..d96c3945 100644 --- a/cloudinit/cmd/tests/test_query.py +++ b/cloudinit/cmd/tests/test_query.py @@ -75,6 +75,40 @@ class TestQuery: assert 'usage: query' in out assert 1 == m_cli_log.call_count + @pytest.mark.parametrize( + "inst_data,varname,expected_error", ( + ( + '{"v1": {"key-2": "value-2"}}', + 'v1.absent_leaf', + "instance-data 'v1' has no 'absent_leaf'\n" + ), + ( + '{"v1": {"key-2": "value-2"}}', + 'absent_key', + "Undefined instance-data key 'absent_key'\n" + ), + ) + ) + def test_handle_args_error_on_invalid_vaname_paths( + self, inst_data, varname, expected_error, caplog, tmpdir + ): + """Error when varname is not a valid instance-data variable path.""" + instance_data = tmpdir.join('instance-data') + instance_data.write(inst_data) + args = self.args( + debug=False, dump_all=False, format=None, + instance_data=instance_data.strpath, + list_keys=False, user_data=None, vendor_data=None, varname=varname + ) + paths, _, _, _ = self._setup_paths(tmpdir) + with mock.patch('cloudinit.cmd.query.read_cfg_paths') as m_paths: + m_paths.return_value = paths + with mock.patch( + "cloudinit.cmd.query.addLogHandlerCLI", return_value="" + ): + assert 1 == query.handle_args('anyname', args) + assert expected_error in caplog.text + def test_handle_args_error_on_missing_instance_data(self, caplog, tmpdir): """When instance_data file path does not exist, log an error.""" absent_fn = tmpdir.join('absent') @@ -166,7 +200,7 @@ class TestQuery: assert 0 == query.handle_args('anyname', args) out, _err = capsys.readouterr() cmd_output = json.loads(out) - assert "it worked" == cmd_output['my_var'] + assert "it worked" == cmd_output['my-var'] if ud_expected == "ci-b64:": ud_expected = "ci-b64:{}".format(b64e(ud_src)) if vd_expected == "ci-b64:": @@ -193,8 +227,8 @@ class TestQuery: m_getuid.return_value = 0 assert 0 == query.handle_args('anyname', args) expected = ( - '{\n "my_var": "it worked",\n "userdata": "ud",\n ' - '"vendordata": "vd"\n}\n' + '{\n "my-var": "it worked",\n ' + '"userdata": "ud",\n "vendordata": "vd"\n}\n' ) out, _err = capsys.readouterr() assert expected == out @@ -211,7 +245,7 @@ class TestQuery: m_getuid.return_value = 100 assert 0 == query.handle_args('anyname', args) expected = ( - '{\n "my_var": "it worked",\n "userdata": "<%s> file:ud",\n' + '{\n "my-var": "it worked",\n "userdata": "<%s> file:ud",\n' ' "vendordata": "<%s> file:vd"\n}\n' % ( REDACT_SENSITIVE_VALUE, REDACT_SENSITIVE_VALUE ) @@ -233,21 +267,38 @@ class TestQuery: out, _err = capsys.readouterr() assert 'it worked\n' == out - def test_handle_args_returns_nested_varname(self, capsys, tmpdir): + @pytest.mark.parametrize( + 'inst_data,varname,expected', + ( + ( + '{"v1": {"key-2": "value-2"}, "my-var": "it worked"}', + 'v1.key_2', + 'value-2\n' + ), + # Assert no jinja underscore-delimited aliases are reported on CLI + ( + '{"v1": {"something-hyphenated": {"no.underscores":"x",' + ' "no-alias": "y"}}, "my-var": "it worked"}', + 'v1.something_hyphenated', + '{\n "no-alias": "y",\n "no.underscores": "x"\n}\n' + ), + ) + ) + def test_handle_args_returns_nested_varname( + self, inst_data, varname, expected, capsys, tmpdir + ): """If user_data file is a jinja template render instance-data vars.""" instance_data = tmpdir.join('instance-data') - instance_data.write( - '{"v1": {"key-2": "value-2"}, "my-var": "it worked"}' - ) + instance_data.write(inst_data) args = self.args( debug=False, dump_all=False, format=None, instance_data=instance_data.strpath, user_data='ud', - vendor_data='vd', list_keys=False, varname='v1.key_2') + vendor_data='vd', list_keys=False, varname=varname) with mock.patch('os.getuid') as m_getuid: m_getuid.return_value = 100 assert 0 == query.handle_args('anyname', args) out, _err = capsys.readouterr() - assert 'value-2\n' == out + assert expected == out def test_handle_args_returns_standardized_vars_to_top_level_aliases( self, capsys, tmpdir diff --git a/cloudinit/handlers/jinja_template.py b/cloudinit/handlers/jinja_template.py index 5033abbb..de88a5ea 100644 --- a/cloudinit/handlers/jinja_template.py +++ b/cloudinit/handlers/jinja_template.py @@ -1,14 +1,18 @@ # This file is part of cloud-init. See LICENSE file for license information. +import copy from errno import EACCES import os import re +from typing import Optional try: from jinja2.exceptions import UndefinedError as JUndefinedError + from jinja2.lexer import operator_re except ImportError: # No jinja2 dependency JUndefinedError = Exception + operator_re = re.compile(r'[-.]') from cloudinit import handlers from cloudinit import log as logging @@ -97,7 +101,9 @@ def render_jinja_payload_from_file( def render_jinja_payload(payload, payload_fn, instance_data, debug=False): instance_jinja_vars = convert_jinja_instance_data( instance_data, - decode_paths=instance_data.get('base64-encoded-keys', [])) + decode_paths=instance_data.get('base64-encoded-keys', []), + include_key_aliases=True + ) if debug: LOG.debug('Converted jinja variables\n%s', json_dumps(instance_jinja_vars)) @@ -118,7 +124,30 @@ def render_jinja_payload(payload, payload_fn, instance_data, debug=False): return rendered_payload -def convert_jinja_instance_data(data, prefix='', sep='/', decode_paths=()): +def get_jinja_variable_alias(orig_name: str) -> Optional[str]: + """Return a jinja variable alias, replacing any operators with underscores. + + Provide underscore-delimited key aliases to simplify dot-notation + attribute references for keys which contain operators "." or "-". + This provides for simpler short-hand jinja attribute notation + allowing one to avoid quoting keys which contain operators. + {{ ds.v1_0.config.user_network_config }} instead of + {{ ds['v1.0'].config["user.network-config"] }}. + + :param orig_name: String representing a jinja variable name to scrub/alias. + + :return: A string with any jinja operators replaced if needed. Otherwise, + none if no alias required. + """ + alias_name = re.sub(operator_re, '_', orig_name) + if alias_name != orig_name: + return alias_name + return None + + +def convert_jinja_instance_data( + data, prefix='', sep='/', decode_paths=(), include_key_aliases=False +): """Process instance-data.json dict for use in jinja templates. Replace hyphens with underscores for jinja templates and decode any @@ -127,21 +156,24 @@ def convert_jinja_instance_data(data, prefix='', sep='/', decode_paths=()): result = {} decode_paths = [path.replace('-', '_') for path in decode_paths] for key, value in sorted(data.items()): - if '-' in key: - # Standardize keys for use in #cloud-config/shell templates - key = key.replace('-', '_') key_path = '{0}{1}{2}'.format(prefix, sep, key) if prefix else key if key_path in decode_paths: value = b64d(value) if isinstance(value, dict): result[key] = convert_jinja_instance_data( - value, key_path, sep=sep, decode_paths=decode_paths) - if re.match(r'v\d+', key): + value, key_path, sep=sep, decode_paths=decode_paths, + include_key_aliases=include_key_aliases + ) + if re.match(r'v\d+$', key): # Copy values to top-level aliases for subkey, subvalue in result[key].items(): - result[subkey] = subvalue + result[subkey] = copy.deepcopy(subvalue) else: result[key] = value + if include_key_aliases: + alias_name = get_jinja_variable_alias(key) + if alias_name: + result[alias_name] = copy.deepcopy(result[key]) return result # vi: ts=4 expandtab diff --git a/cloudinit/sources/DataSourceLXD.py b/cloudinit/sources/DataSourceLXD.py index 55ae52a2..469707d2 100644 --- a/cloudinit/sources/DataSourceLXD.py +++ b/cloudinit/sources/DataSourceLXD.py @@ -190,19 +190,16 @@ class DataSourceLXD(sources.DataSource): self.metadata = _raw_instance_data_to_dict( "meta-data", self._crawled_metadata.get("meta-data") ) - if LXD_SOCKET_API_VERSION in self._crawled_metadata: - config = self._crawled_metadata[LXD_SOCKET_API_VERSION].get( - "config", {} + config = self._crawled_metadata.get("config", {}) + user_metadata = config.get("user.meta-data", {}) + if user_metadata: + user_metadata = _raw_instance_data_to_dict( + "user.meta-data", user_metadata + ) + if not isinstance(self.metadata, dict): + self.metadata = util.mergemanydict( + [util.load_yaml(self.metadata), user_metadata] ) - user_metadata = config.get("user.meta-data", {}) - if user_metadata: - user_metadata = _raw_instance_data_to_dict( - "user.meta-data", user_metadata - ) - if not isinstance(self.metadata, dict): - self.metadata = util.mergemanydict( - [util.load_yaml(self.metadata), user_metadata] - ) if "user-data" in self._crawled_metadata: self.userdata_raw = self._crawled_metadata["user-data"] if "network-config" in self._crawled_metadata: @@ -304,7 +301,8 @@ def read_metadata( if metadata_only: return md # Skip network-data, vendor-data, user-data - md[LXD_SOCKET_API_VERSION] = { + md = { + "_metadata_api_version": api_version, # Document API version read "config": {}, "meta-data": md["meta-data"] } @@ -345,7 +343,7 @@ def read_metadata( # Leave raw data values/format unchanged to represent it in # instance-data.json for cloud-init query or jinja template # use. - md[LXD_SOCKET_API_VERSION]["config"][cfg_key] = response.text + md["config"][cfg_key] = response.text # Promote common CONFIG_KEY_ALIASES to top-level keys. if cfg_key in CONFIG_KEY_ALIASES: # Due to sort of config_routes, promote cloud-init.* diff --git a/cloudinit/sources/tests/test_lxd.py b/cloudinit/sources/tests/test_lxd.py index fc2a41df..a6e51f3b 100644 --- a/cloudinit/sources/tests/test_lxd.py +++ b/cloudinit/sources/tests/test_lxd.py @@ -42,15 +42,12 @@ LXD_V1_METADATA = { "network-config": NETWORK_V1, "user-data": "#cloud-config\npackages: [sl]\n", "vendor-data": "#cloud-config\nruncmd: ['echo vendor-data']\n", - "1.0": { - "meta-data": "instance-id: my-lxc\nlocal-hostname: my-lxc\n\n", - "config": { - "user.user-data": - "instance-id: my-lxc\nlocal-hostname: my-lxc\n\n", - "user.vendor-data": - "#cloud-config\nruncmd: ['echo vendor-data']\n", - "user.network-config": yaml.safe_dump(NETWORK_V1), - } + "config": { + "user.user-data": + "instance-id: my-lxc\nlocal-hostname: my-lxc\n\n", + "user.vendor-data": + "#cloud-config\nruncmd: ['echo vendor-data']\n", + "user.network-config": yaml.safe_dump(NETWORK_V1), } } @@ -190,8 +187,10 @@ class TestReadMetadata: "http://lxd/1.0/meta-data": "local-hostname: md\n", "http://lxd/1.0/config": "[]", }, - {"1.0": {"config": {}, "meta-data": "local-hostname: md\n"}, - "meta-data": "local-hostname: md\n"}, + { + "_metadata_api_version": lxd.LXD_SOCKET_API_VERSION, + "config": {}, "meta-data": "local-hostname: md\n" + }, ["[GET] [HTTP:200] http://lxd/1.0/meta-data", "[GET] [HTTP:200] http://lxd/1.0/config"], ), @@ -211,12 +210,10 @@ class TestReadMetadata: "http://lxd/1.0/config/user.vendor-data": "", # 404 }, { - "1.0": { - "config": { - "user.custom1": "custom1", # Not promoted - "user.network-config": "net-config", - }, - "meta-data": "local-hostname: md\n", + "_metadata_api_version": lxd.LXD_SOCKET_API_VERSION, + "config": { + "user.custom1": "custom1", # Not promoted + "user.network-config": "net-config", }, "meta-data": "local-hostname: md\n", "network-config": "net-config", @@ -250,15 +247,13 @@ class TestReadMetadata: "http://lxd/1.0/config/user.vendor-data": "vendor-data", }, { - "1.0": { - "config": { - "user.custom1": "custom1", # Not promoted - "user.meta-data": "meta-data", - "user.network-config": "net-config", - "user.user-data": "user-data", - "user.vendor-data": "vendor-data", - }, - "meta-data": "local-hostname: md\n", + "_metadata_api_version": lxd.LXD_SOCKET_API_VERSION, + "config": { + "user.custom1": "custom1", # Not promoted + "user.meta-data": "meta-data", + "user.network-config": "net-config", + "user.user-data": "user-data", + "user.vendor-data": "vendor-data", }, "meta-data": "local-hostname: md\n", "network-config": "net-config", @@ -303,19 +298,17 @@ class TestReadMetadata: "cloud-init.vendor-data", }, { - "1.0": { - "config": { - "user.meta-data": "user.meta-data", - "user.network-config": "user.network-config", - "user.user-data": "user.user-data", - "user.vendor-data": "user.vendor-data", - "cloud-init.network-config": - "cloud-init.network-config", - "cloud-init.user-data": "cloud-init.user-data", - "cloud-init.vendor-data": - "cloud-init.vendor-data", - }, - "meta-data": "local-hostname: md\n", + "_metadata_api_version": lxd.LXD_SOCKET_API_VERSION, + "config": { + "user.meta-data": "user.meta-data", + "user.network-config": "user.network-config", + "user.user-data": "user.user-data", + "user.vendor-data": "user.vendor-data", + "cloud-init.network-config": + "cloud-init.network-config", + "cloud-init.user-data": "cloud-init.user-data", + "cloud-init.vendor-data": + "cloud-init.vendor-data", }, "meta-data": "local-hostname: md\n", "network-config": "cloud-init.network-config", diff --git a/doc/rtd/topics/instancedata.rst b/doc/rtd/topics/instancedata.rst index 6c17139f..c33b907a 100644 --- a/doc/rtd/topics/instancedata.rst +++ b/doc/rtd/topics/instancedata.rst @@ -530,12 +530,18 @@ Both user-data scripts and **#cloud-config** data support jinja template rendering. When the first line of the provided user-data begins with, **## template: jinja** cloud-init will use jinja to render that file. -Any instance-data-sensitive.json variables are surfaced as dot-delimited -jinja template variables because cloud-config modules are run as 'root' -user. +Any instance-data-sensitive.json variables are surfaced as jinja template +variables because cloud-config modules are run as 'root' user. - -Below are some examples of providing these types of user-data: +.. note:: + cloud-init also provides jinja-safe key aliases for any instance-data.json + keys which contain jinja operator characters such as +, -, ., /, etc. Any + jinja operator will be replaced with underscores in the jinja-safe key + alias. This allows for cloud-init templates to use aliased variable + references which allow for jinja's dot-notation reference such as + ``{{ ds.v1_0.my_safe_key }}`` instead of ``{{ ds["v1.0"]["my/safe-key"] }}``. + +Below are some other examples of using jinja templates in user-data: * Cloud config calling home with the ec2 public hostname and availability-zone diff --git a/tests/integration_tests/datasources/test_lxd_discovery.py b/tests/integration_tests/datasources/test_lxd_discovery.py index 93200962..3f05e906 100644 --- a/tests/integration_tests/datasources/test_lxd_discovery.py +++ b/tests/integration_tests/datasources/test_lxd_discovery.py @@ -53,7 +53,9 @@ def test_lxd_datasource_discovery(client: IntegrationInstance): assert "lxd" == v1["platform"] assert "LXD socket API v. 1.0 (/dev/lxd/sock)" == v1["subplatform"] ds_cfg = json.loads(client.execute('cloud-init query ds').stdout) - assert ["config", "meta_data"] == sorted(list(ds_cfg["1.0"].keys())) + assert ["_doc", "_metadata_api_version", "config", "meta-data"] == sorted( + list(ds_cfg.keys()) + ) if ( client.settings.PLATFORM == "lxd_vm" and ImageSpecification.from_os_image().release in ("xenial", "bionic") @@ -62,15 +64,18 @@ def test_lxd_datasource_discovery(client: IntegrationInstance): # to start the lxd-agent. # https://github.com/canonical/pycloudlib/blob/main/pycloudlib/\ # lxd/defaults.py#L13-L27 - lxd_config_keys = ["user.meta_data", "user.vendor_data"] + # Underscore-delimited aliases exist for any keys containing hyphens or + # dots. + lxd_config_keys = ["user.meta-data", "user.vendor-data"] else: - lxd_config_keys = ["user.meta_data"] - assert lxd_config_keys == list(ds_cfg["1.0"]["config"].keys()) + lxd_config_keys = ["user.meta-data"] + assert "1.0" == ds_cfg["_metadata_api_version"] + assert lxd_config_keys == list(ds_cfg["config"].keys()) assert {"public-keys": v1["public_ssh_keys"][0]} == ( - yaml.safe_load(ds_cfg["1.0"]["config"]["user.meta_data"]) + yaml.safe_load(ds_cfg["config"]["user.meta-data"]) ) assert ( - "#cloud-config\ninstance-id" in ds_cfg["1.0"]["meta_data"] + "#cloud-config\ninstance-id" in ds_cfg["meta-data"] ) # Assert NoCloud seed data is still present in cloud image metadata # This will start failing if we redact metadata templates from diff --git a/tests/integration_tests/modules/test_jinja_templating.py b/tests/integration_tests/modules/test_jinja_templating.py index 35b8ee2d..fe8eff1a 100644 --- a/tests/integration_tests/modules/test_jinja_templating.py +++ b/tests/integration_tests/modules/test_jinja_templating.py @@ -11,6 +11,7 @@ USER_DATA = """\ runcmd: - echo {{v1.local_hostname}} > /var/tmp/runcmd_output - echo {{merged_cfg._doc}} >> /var/tmp/runcmd_output + - echo {{v1['local-hostname']}} >> /var/tmp/runcmd_output """ @@ -18,13 +19,16 @@ runcmd: def test_runcmd_with_variable_substitution(client: IntegrationInstance): """Test jinja substitution. - Ensure we can also substitute variables from instance-data-sensitive - LP: #1931392 + Ensure underscore-delimited aliases exist for hyphenated key and + we can also substitute variables from instance-data-sensitive + LP: #1931392. """ + hostname = client.execute('hostname').stdout.strip() expected = [ - client.execute('hostname').stdout.strip(), + hostname, ('Merged cloud-init system config from /etc/cloud/cloud.cfg and ' - '/etc/cloud/cloud.cfg.d/') + '/etc/cloud/cloud.cfg.d/'), + hostname ] output = client.read_from_file('/var/tmp/runcmd_output') verify_ordered_items_in_text(expected, output) diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 30293e9e..230866b9 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -5,6 +5,7 @@ import copy import errno import os +import pytest import shutil import tempfile from textwrap import dedent @@ -281,17 +282,44 @@ class TestJinjaTemplatePartHandler(CiTestCase): self.logs.getvalue()) -class TestConvertJinjaInstanceData(CiTestCase): - - def test_convert_instance_data_hyphens_to_underscores(self): - """Replace hyphenated keys with underscores in instance-data.""" - data = {'hyphenated-key': 'hyphenated-val', - 'underscore_delim_key': 'underscore_delimited_val'} - expected_data = {'hyphenated_key': 'hyphenated-val', - 'underscore_delim_key': 'underscore_delimited_val'} - self.assertEqual( - expected_data, - convert_jinja_instance_data(data=data)) +class TestConvertJinjaInstanceData: + + @pytest.mark.parametrize( + "include_key_aliases,data,expected", ( + ( + False, + {'my-key': 'my-val'}, + {'my-key': 'my-val'} + ), + ( + True, + {'my-key': 'my-val'}, + {'my-key': 'my-val', 'my_key': 'my-val'} + ), + ( + False, + {'my.key': 'my.val'}, + {'my.key': 'my.val'} + ), + ( + True, + {'my.key': 'my.val'}, + {'my.key': 'my.val', 'my_key': 'my.val'} + ), + ( + True, + {'my/key': 'my/val'}, + {'my/key': 'my/val', 'my_key': 'my/val'} + ), + ) + ) + def test_convert_instance_data_operators_to_underscores( + self, include_key_aliases, data, expected + ): + """Replace Jinja operators keys with underscores in instance-data.""" + assert expected == convert_jinja_instance_data( + data=data, include_key_aliases=include_key_aliases + ) def test_convert_instance_data_promotes_versioned_keys_to_top_level(self): """Any versioned keys are promoted as top-level keys @@ -307,11 +335,10 @@ class TestConvertJinjaInstanceData(CiTestCase): expected_data.update({'v1key1': 'v1.1', 'v2key1': 'v2.1'}) converted_data = convert_jinja_instance_data(data=data) - self.assertCountEqual( - ['ds', 'v1', 'v2', 'v1key1', 'v2key1'], converted_data.keys()) - self.assertEqual( - expected_data, - converted_data) + assert sorted(['ds', 'v1', 'v2', 'v1key1', 'v2key1']) == sorted( + converted_data.keys() + ) + assert expected_data == converted_data def test_convert_instance_data_most_recent_version_of_promoted_keys(self): """The most-recent versioned key value is promoted to top-level.""" @@ -324,9 +351,7 @@ class TestConvertJinjaInstanceData(CiTestCase): 'key3': 'newer v2 key3'}) converted_data = convert_jinja_instance_data(data=data) - self.assertEqual( - expected_data, - converted_data) + assert expected_data == converted_data def test_convert_instance_data_decodes_decode_paths(self): """Any decode_paths provided are decoded by convert_instance_data.""" @@ -336,9 +361,7 @@ class TestConvertJinjaInstanceData(CiTestCase): converted_data = convert_jinja_instance_data( data=data, decode_paths=('key1/subkey1',)) - self.assertEqual( - expected_data, - converted_data) + assert expected_data == converted_data class TestRenderJinjaPayload(CiTestCase): @@ -355,6 +378,7 @@ class TestRenderJinjaPayload(CiTestCase): DEBUG: Converted jinja variables { "hostname": "foo", + "instance-id": "iid", "instance_id": "iid", "v1": { "hostname": "foo" |