jinja: provide and document jinja-safe key aliases in instance-data (SC-622) (#1123)

Allow #cloud-config and cloud-init query to use underscore-delimited
"jinja-safe" key aliases for any instance-data.json keys
containing jinja operator characters.

This provides a means to use Jinja's dot-notation instead of square brackets
and quoting to reference "unsafe" obtain attribute names.

Support for these aliased keys is available to both #cloud-config user-data and
`cloud-init query`.

For example #cloud-config alias access can look like:
  {{ ds.config.user_network_config }}

  - instead of -

  {{ ds.config["user.network-config"] }}

3 files changed, 65 insertions, 32 deletions

diff --git a/tests/integration_tests/datasources/test_lxd_discovery.py b/tests/integration_tests/datasources/test_lxd_discovery.py
index 93200962..3f05e906 100644
--- a/tests/integration_tests/datasources/test_lxd_discovery.py
+++ b/tests/integration_tests/datasources/test_lxd_discovery.py
@@ -53,7 +53,9 @@ def test_lxd_datasource_discovery(client: IntegrationInstance):
     assert "lxd" == v1["platform"]
     assert "LXD socket API v. 1.0 (/dev/lxd/sock)" == v1["subplatform"]
     ds_cfg = json.loads(client.execute('cloud-init query ds').stdout)
-    assert ["config", "meta_data"] == sorted(list(ds_cfg["1.0"].keys()))
+    assert ["_doc", "_metadata_api_version", "config", "meta-data"] == sorted(
+        list(ds_cfg.keys())
+    )
     if (
         client.settings.PLATFORM == "lxd_vm" and
         ImageSpecification.from_os_image().release in ("xenial", "bionic")
@@ -62,15 +64,18 @@ def test_lxd_datasource_discovery(client: IntegrationInstance):
         # to start the lxd-agent.
         # https://github.com/canonical/pycloudlib/blob/main/pycloudlib/\
         #    lxd/defaults.py#L13-L27
-        lxd_config_keys = ["user.meta_data", "user.vendor_data"]
+        # Underscore-delimited aliases exist for any keys containing hyphens or
+        # dots.
+        lxd_config_keys = ["user.meta-data", "user.vendor-data"]
     else:
-        lxd_config_keys = ["user.meta_data"]
-    assert lxd_config_keys == list(ds_cfg["1.0"]["config"].keys())
+        lxd_config_keys = ["user.meta-data"]
+    assert "1.0" == ds_cfg["_metadata_api_version"]
+    assert lxd_config_keys == list(ds_cfg["config"].keys())
     assert {"public-keys": v1["public_ssh_keys"][0]} == (
-        yaml.safe_load(ds_cfg["1.0"]["config"]["user.meta_data"])
+        yaml.safe_load(ds_cfg["config"]["user.meta-data"])
     )
     assert (
-        "#cloud-config\ninstance-id" in ds_cfg["1.0"]["meta_data"]
+        "#cloud-config\ninstance-id" in ds_cfg["meta-data"]
     )
     # Assert NoCloud seed data is still present in cloud image metadata
     # This will start failing if we redact metadata templates from
diff --git a/tests/integration_tests/modules/test_jinja_templating.py b/tests/integration_tests/modules/test_jinja_templating.py
index 35b8ee2d..fe8eff1a 100644
--- a/tests/integration_tests/modules/test_jinja_templating.py
+++ b/tests/integration_tests/modules/test_jinja_templating.py
@@ -11,6 +11,7 @@ USER_DATA = """\
 runcmd:
   - echo {{v1.local_hostname}} > /var/tmp/runcmd_output
   - echo {{merged_cfg._doc}} >> /var/tmp/runcmd_output
+  - echo {{v1['local-hostname']}} >> /var/tmp/runcmd_output
 """
 
 
@@ -18,13 +19,16 @@ runcmd:
 def test_runcmd_with_variable_substitution(client: IntegrationInstance):
     """Test jinja substitution.
 
-    Ensure we can also substitute variables from instance-data-sensitive
-    LP: #1931392
+    Ensure underscore-delimited aliases exist for hyphenated key and
+    we can also substitute variables from instance-data-sensitive
+    LP: #1931392.
     """
+    hostname = client.execute('hostname').stdout.strip()
     expected = [
-        client.execute('hostname').stdout.strip(),
+        hostname,
         ('Merged cloud-init system config from /etc/cloud/cloud.cfg and '
-            '/etc/cloud/cloud.cfg.d/')
+            '/etc/cloud/cloud.cfg.d/'),
+        hostname
     ]
     output = client.read_from_file('/var/tmp/runcmd_output')
     verify_ordered_items_in_text(expected, output)
diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py
index 30293e9e..230866b9 100644
--- a/tests/unittests/test_builtin_handlers.py
+++ b/tests/unittests/test_builtin_handlers.py
@@ -5,6 +5,7 @@
 import copy
 import errno
 import os
+import pytest
 import shutil
 import tempfile
 from textwrap import dedent
@@ -281,17 +282,44 @@ class TestJinjaTemplatePartHandler(CiTestCase):
             self.logs.getvalue())
 
 
-class TestConvertJinjaInstanceData(CiTestCase):
-
-    def test_convert_instance_data_hyphens_to_underscores(self):
-        """Replace hyphenated keys with underscores in instance-data."""
-        data = {'hyphenated-key': 'hyphenated-val',
-                'underscore_delim_key': 'underscore_delimited_val'}
-        expected_data = {'hyphenated_key': 'hyphenated-val',
-                         'underscore_delim_key': 'underscore_delimited_val'}
-        self.assertEqual(
-            expected_data,
-            convert_jinja_instance_data(data=data))
+class TestConvertJinjaInstanceData:
+
+    @pytest.mark.parametrize(
+        "include_key_aliases,data,expected", (
+            (
+                False,
+                {'my-key': 'my-val'},
+                {'my-key': 'my-val'}
+            ),
+            (
+                True,
+                {'my-key': 'my-val'},
+                {'my-key': 'my-val', 'my_key': 'my-val'}
+            ),
+            (
+                False,
+                {'my.key': 'my.val'},
+                {'my.key': 'my.val'}
+            ),
+            (
+                True,
+                {'my.key': 'my.val'},
+                {'my.key': 'my.val', 'my_key': 'my.val'}
+            ),
+            (
+                True,
+                {'my/key': 'my/val'},
+                {'my/key': 'my/val', 'my_key': 'my/val'}
+            ),
+        )
+    )
+    def test_convert_instance_data_operators_to_underscores(
+        self, include_key_aliases, data, expected
+    ):
+        """Replace Jinja operators keys with underscores in instance-data."""
+        assert expected == convert_jinja_instance_data(
+            data=data, include_key_aliases=include_key_aliases
+        )
 
     def test_convert_instance_data_promotes_versioned_keys_to_top_level(self):
         """Any versioned keys are promoted as top-level keys
@@ -307,11 +335,10 @@ class TestConvertJinjaInstanceData(CiTestCase):
         expected_data.update({'v1key1': 'v1.1', 'v2key1': 'v2.1'})
 
         converted_data = convert_jinja_instance_data(data=data)
-        self.assertCountEqual(
-            ['ds', 'v1', 'v2', 'v1key1', 'v2key1'], converted_data.keys())
-        self.assertEqual(
-            expected_data,
-            converted_data)
+        assert sorted(['ds', 'v1', 'v2', 'v1key1', 'v2key1']) == sorted(
+            converted_data.keys()
+        )
+        assert expected_data == converted_data
 
     def test_convert_instance_data_most_recent_version_of_promoted_keys(self):
         """The most-recent versioned key value is promoted to top-level."""
@@ -324,9 +351,7 @@ class TestConvertJinjaInstanceData(CiTestCase):
              'key3': 'newer v2 key3'})
 
         converted_data = convert_jinja_instance_data(data=data)
-        self.assertEqual(
-            expected_data,
-            converted_data)
+        assert expected_data == converted_data
 
     def test_convert_instance_data_decodes_decode_paths(self):
         """Any decode_paths provided are decoded by convert_instance_data."""
@@ -336,9 +361,7 @@ class TestConvertJinjaInstanceData(CiTestCase):
 
         converted_data = convert_jinja_instance_data(
             data=data, decode_paths=('key1/subkey1',))
-        self.assertEqual(
-            expected_data,
-            converted_data)
+        assert expected_data == converted_data
 
 
 class TestRenderJinjaPayload(CiTestCase):
@@ -355,6 +378,7 @@ class TestRenderJinjaPayload(CiTestCase):
             DEBUG: Converted jinja variables
             {
              "hostname": "foo",
+             "instance-id": "iid",
              "instance_id": "iid",
              "v1": {
               "hostname": "foo"