instance-data: add cloud-init merged_cfg and sys_info keys to json (#214)

Cloud-config userdata provided as jinja templates are now distro,
platform and merged cloud config aware. The cloud-init query command
will also surface this config data.

Now users can selectively render portions of cloud-config based on:
* distro name, version, release
* python version
* merged cloud config values
* machine platform
* kernel

To support template handling of this config, add new top-level
keys to /run/cloud-init/instance-data.json.

The new 'merged_cfg' key represents merged cloud config from
/etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/*.

The new 'sys_info' key which captures distro and platform
info from cloudinit.util.system_info.

Cloud config userdata templates can render conditional content
based on these additional environmental checks such as the following
simple example:

```
  ## template: jinja
  #cloud-config
   runcmd:
  {% if distro == 'opensuse' %}
    - sh /custom-setup-sles
  {% elif distro == 'centos' %}
    - sh /custom-setup-centos
  {% elif distro == 'debian' %}
    - sh /custom-setup-debian
  {% endif %}
```

To see all values: sudo cloud-init query --all

Any keys added to the standardized v1 keys are guaranteed to not
change or drop on future released of cloud-init. 'v1' keys will be retained
for backward-compatibility even if a new standardized 'v2' set of keys
are introduced

The following standardized v1 keys are added:
* distro, distro_release, distro_version, kernel_version, machine,
python_version, system_platform, variant

LP: #1865969

1 files changed, 52 insertions, 3 deletions

diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index fd12d87b..5976e234 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -172,9 +172,7 @@ class CloudTestCase(unittest2.TestCase):
                 'Skipping instance-data.json test.'
                 ' OS: %s not bionic or newer' % self.os_name)
         instance_data = json.loads(out)
-        self.assertItemsEqual(
-            [],
-            instance_data['base64_encoded_keys'])
+        self.assertItemsEqual(['ci_cfg'], instance_data['sensitive_keys'])
         ds = instance_data.get('ds', {})
         v1_data = instance_data.get('v1', {})
         metadata = ds.get('meta-data', {})
@@ -201,6 +199,23 @@ class CloudTestCase(unittest2.TestCase):
         self.assertIn('i-', v1_data['instance_id'])
         self.assertIn('ip-', v1_data['local_hostname'])
         self.assertIsNotNone(v1_data['region'], 'expected ec2 region')
+        self.assertIsNotNone(
+            re.match(r'\d\.\d+\.\d+-\d+-aws', v1_data['kernel_release']))
+        self.assertEqual(
+            'redacted for non-root user', instance_data['merged_cfg'])
+        self.assertEqual(self.os_cfg['os'], v1_data['variant'])
+        self.assertEqual(self.os_cfg['os'], v1_data['distro'])
+        self.assertEqual(
+            self.os_cfg['os'], instance_data["sys_info"]['dist'][0],
+            "Unexpected sys_info dist value")
+        self.assertEqual(self.os_name, v1_data['distro_release'])
+        self.assertEqual(
+            str(self.os_cfg['version']), v1_data['distro_version'])
+        self.assertEqual('x86_64', v1_data['machine'])
+        self.assertIsNotNone(
+            re.match(r'3.\d\.\d', v1_data['python_version']),
+            "unexpected python version: {ver}".format(
+                ver=v1_data["python_version"]))
 
     def test_instance_data_json_lxd(self):
         """Validate instance-data.json content by lxd platform.
@@ -237,6 +252,23 @@ class CloudTestCase(unittest2.TestCase):
         self.assertIsNone(
             v1_data['region'],
             'found unexpected lxd region %s' % v1_data['region'])
+        self.assertIsNotNone(
+            re.match(r'\d\.\d+\.\d+-\d+', v1_data['kernel_release']))
+        self.assertEqual(
+            'redacted for non-root user', instance_data['merged_cfg'])
+        self.assertEqual(self.os_cfg['os'], v1_data['variant'])
+        self.assertEqual(self.os_cfg['os'], v1_data['distro'])
+        self.assertEqual(
+            self.os_cfg['os'], instance_data["sys_info"]['dist'][0],
+            "Unexpected sys_info dist value")
+        self.assertEqual(self.os_name, v1_data['distro_release'])
+        self.assertEqual(
+            str(self.os_cfg['version']), v1_data['distro_version'])
+        self.assertEqual('x86_64', v1_data['machine'])
+        self.assertIsNotNone(
+            re.match(r'3.\d\.\d', v1_data['python_version']),
+            "unexpected python version: {ver}".format(
+                ver=v1_data["python_version"]))
 
     def test_instance_data_json_kvm(self):
         """Validate instance-data.json content by nocloud-kvm platform.
@@ -278,6 +310,23 @@ class CloudTestCase(unittest2.TestCase):
         self.assertIsNone(
             v1_data['region'],
             'found unexpected lxd region %s' % v1_data['region'])
+        self.assertIsNotNone(
+            re.match(r'\d\.\d+\.\d+-\d+', v1_data['kernel_release']))
+        self.assertEqual(
+            'redacted for non-root user', instance_data['merged_cfg'])
+        self.assertEqual(self.os_cfg['os'], v1_data['variant'])
+        self.assertEqual(self.os_cfg['os'], v1_data['distro'])
+        self.assertEqual(
+            self.os_cfg['os'], instance_data["sys_info"]['dist'][0],
+            "Unexpected sys_info dist value")
+        self.assertEqual(self.os_name, v1_data['distro_release'])
+        self.assertEqual(
+                str(self.os_cfg['version']), v1_data['distro_version'])
+        self.assertEqual('x86_64', v1_data['machine'])
+        self.assertIsNotNone(
+            re.match(r'3.\d\.\d', v1_data['python_version']),
+            "unexpected python version: {ver}".format(
+                ver=v1_data["python_version"]))
 
 
 class PasswordListTest(CloudTestCase):