instance-data: add cloud-init merged_cfg and sys_info keys to json (#214)

Cloud-config userdata provided as jinja templates are now distro,
platform and merged cloud config aware. The cloud-init query command
will also surface this config data.

Now users can selectively render portions of cloud-config based on:
* distro name, version, release
* python version
* merged cloud config values
* machine platform
* kernel

To support template handling of this config, add new top-level
keys to /run/cloud-init/instance-data.json.

The new 'merged_cfg' key represents merged cloud config from
/etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/*.

The new 'sys_info' key which captures distro and platform
info from cloudinit.util.system_info.

Cloud config userdata templates can render conditional content
based on these additional environmental checks such as the following
simple example:

```
  ## template: jinja
  #cloud-config
   runcmd:
  {% if distro == 'opensuse' %}
    - sh /custom-setup-sles
  {% elif distro == 'centos' %}
    - sh /custom-setup-centos
  {% elif distro == 'debian' %}
    - sh /custom-setup-debian
  {% endif %}
```

To see all values: sudo cloud-init query --all

Any keys added to the standardized v1 keys are guaranteed to not
change or drop on future released of cloud-init. 'v1' keys will be retained
for backward-compatibility even if a new standardized 'v2' set of keys
are introduced

The following standardized v1 keys are added:
* distro, distro_release, distro_version, kernel_version, machine,
python_version, system_platform, variant

LP: #1865969

6 files changed, 426 insertions, 138 deletions

diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 805d803d..a6e6d202 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -89,26 +89,26 @@ def process_instance_metadata(metadata, key_path='', sensitive_keys=()):
     @return Dict copy of processed metadata.
     """
     md_copy = copy.deepcopy(metadata)
-    md_copy['base64_encoded_keys'] = []
-    md_copy['sensitive_keys'] = []
+    base64_encoded_keys = []
+    sens_keys = []
     for key, val in metadata.items():
         if key_path:
             sub_key_path = key_path + '/' + key
         else:
             sub_key_path = key
         if key in sensitive_keys or sub_key_path in sensitive_keys:
-            md_copy['sensitive_keys'].append(sub_key_path)
+            sens_keys.append(sub_key_path)
         if isinstance(val, str) and val.startswith('ci-b64:'):
-            md_copy['base64_encoded_keys'].append(sub_key_path)
+            base64_encoded_keys.append(sub_key_path)
             md_copy[key] = val.replace('ci-b64:', '')
         if isinstance(val, dict):
             return_val = process_instance_metadata(
                 val, sub_key_path, sensitive_keys)
-            md_copy['base64_encoded_keys'].extend(
-                return_val.pop('base64_encoded_keys'))
-            md_copy['sensitive_keys'].extend(
-                return_val.pop('sensitive_keys'))
+            base64_encoded_keys.extend(return_val.pop('base64_encoded_keys'))
+            sens_keys.extend(return_val.pop('sensitive_keys'))
             md_copy[key] = return_val
+    md_copy['base64_encoded_keys'] = sorted(base64_encoded_keys)
+    md_copy['sensitive_keys'] = sorted(sens_keys)
     return md_copy
 
 
@@ -193,7 +193,7 @@ class DataSource(metaclass=abc.ABCMeta):
 
     # N-tuple of keypaths or keynames redact from instance-data.json for
     # non-root users
-    sensitive_metadata_keys = ('security-credentials',)
+    sensitive_metadata_keys = ('merged_cfg', 'security-credentials',)
 
     def __init__(self, sys_cfg, distro, paths, ud_proc=None):
         self.sys_cfg = sys_cfg
@@ -218,14 +218,15 @@ class DataSource(metaclass=abc.ABCMeta):
     def __str__(self):
         return type_utils.obj_name(self)
 
-    def _get_standardized_metadata(self):
+    def _get_standardized_metadata(self, instance_data):
         """Return a dictionary of standardized metadata keys."""
         local_hostname = self.get_hostname()
         instance_id = self.get_instance_id()
         availability_zone = self.availability_zone
         # In the event of upgrade from existing cloudinit, pickled datasource
         # will not contain these new class attributes. So we need to recrawl
-        # metadata to discover that content.
+        # metadata to discover that content
+        sysinfo = instance_data["sys_info"]
         return {
             'v1': {
                 '_beta_keys': ['subplatform'],
@@ -233,14 +234,22 @@ class DataSource(metaclass=abc.ABCMeta):
                 'availability_zone': availability_zone,
                 'cloud-name': self.cloud_name,
                 'cloud_name': self.cloud_name,
+                'distro': sysinfo["dist"][0],
+                'distro_version': sysinfo["dist"][1],
+                'distro_release': sysinfo["dist"][2],
                 'platform': self.platform_type,
                 'public_ssh_keys': self.get_public_ssh_keys(),
+                'python_version': sysinfo["python"],
                 'instance-id': instance_id,
                 'instance_id': instance_id,
+                'kernel_release': sysinfo["uname"][2],
                 'local-hostname': local_hostname,
                 'local_hostname': local_hostname,
+                'machine': sysinfo["uname"][4],
                 'region': self.region,
-                'subplatform': self.subplatform}}
+                'subplatform': self.subplatform,
+                'system_platform': sysinfo["platform"],
+                'variant': sysinfo["variant"]}}
 
     def clear_cached_attrs(self, attr_defaults=()):
         """Reset any cached metadata attributes to datasource defaults.
@@ -299,9 +308,15 @@ class DataSource(metaclass=abc.ABCMeta):
                 ec2_metadata = getattr(self, 'ec2_metadata')
                 if ec2_metadata != UNSET:
                     instance_data['ds']['ec2_metadata'] = ec2_metadata
-        instance_data.update(
-            self._get_standardized_metadata())
         instance_data['ds']['_doc'] = EXPERIMENTAL_TEXT
+        # Add merged cloud.cfg and sys info for jinja templates and cli query
+        instance_data['merged_cfg'] = copy.deepcopy(self.sys_cfg)
+        instance_data['merged_cfg']['_doc'] = (
+            'Merged cloud-init system config from /etc/cloud/cloud.cfg and'
+            ' /etc/cloud/cloud.cfg.d/')
+        instance_data['sys_info'] = util.system_info()
+        instance_data.update(
+            self._get_standardized_metadata(instance_data))
         try:
             # Process content base64encoding unserializable values
             content = util.json_dumps(instance_data)
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index 6db127e7..541cbbeb 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -55,6 +55,7 @@ class InvalidDataSourceTestSubclassNet(DataSource):
 class TestDataSource(CiTestCase):
 
     with_logs = True
+    maxDiff = None
 
     def setUp(self):
         super(TestDataSource, self).setUp()
@@ -288,27 +289,47 @@ class TestDataSource(CiTestCase):
         tmp = self.tmp_dir()
         datasource = DataSourceTestSubclassNet(
             self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
-        datasource.get_data()
+        sys_info = {
+            "python": "3.7",
+            "platform":
+                "Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal",
+            "uname": ["Linux", "myhost", "5.4.0-24-generic", "SMP blah",
+                      "x86_64"],
+            "variant": "ubuntu", "dist": ["ubuntu", "20.04", "focal"]}
+        with mock.patch("cloudinit.util.system_info", return_value=sys_info):
+            datasource.get_data()
         json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
         content = util.load_file(json_file)
         expected = {
             'base64_encoded_keys': [],
-            'sensitive_keys': [],
+            'merged_cfg': REDACT_SENSITIVE_VALUE,
+            'sensitive_keys': ['merged_cfg'],
+            'sys_info': sys_info,
             'v1': {
                 '_beta_keys': ['subplatform'],
                 'availability-zone': 'myaz',
                 'availability_zone': 'myaz',
                 'cloud-name': 'subclasscloudname',
                 'cloud_name': 'subclasscloudname',
+                'distro': 'ubuntu',
+                'distro_release': 'focal',
+                'distro_version': '20.04',
                 'instance-id': 'iid-datasource',
                 'instance_id': 'iid-datasource',
                 'local-hostname': 'test-subclass-hostname',
                 'local_hostname': 'test-subclass-hostname',
+                'kernel_release': '5.4.0-24-generic',
+                'machine': 'x86_64',
                 'platform': 'mytestsubclass',
                 'public_ssh_keys': [],
+                'python_version': '3.7',
                 'region': 'myregion',
-                'subplatform': 'unknown'},
+                'system_platform':
+                    'Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal',
+                'subplatform': 'unknown',
+                'variant': 'ubuntu'},
             'ds': {
+
                 '_doc': EXPERIMENTAL_TEXT,
                 'meta_data': {'availability_zone': 'myaz',
                               'local-hostname': 'test-subclass-hostname',
@@ -329,28 +350,49 @@ class TestDataSource(CiTestCase):
                 'region': 'myregion',
                 'some': {'security-credentials': {
                     'cred1': 'sekret', 'cred2': 'othersekret'}}})
-        self.assertEqual(
-            ('security-credentials',), datasource.sensitive_metadata_keys)
-        datasource.get_data()
+        self.assertItemsEqual(
+            ('merged_cfg', 'security-credentials',),
+            datasource.sensitive_metadata_keys)
+        sys_info = {
+            "python": "3.7",
+            "platform":
+                "Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal",
+            "uname": ["Linux", "myhost", "5.4.0-24-generic", "SMP blah",
+                      "x86_64"],
+            "variant": "ubuntu", "dist": ["ubuntu", "20.04", "focal"]}
+        with mock.patch("cloudinit.util.system_info", return_value=sys_info):
+            datasource.get_data()
         json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp)
         redacted = util.load_json(util.load_file(json_file))
         expected = {
             'base64_encoded_keys': [],
-            'sensitive_keys': ['ds/meta_data/some/security-credentials'],
+            'merged_cfg': REDACT_SENSITIVE_VALUE,
+            'sensitive_keys': [
+                'ds/meta_data/some/security-credentials', 'merged_cfg'],
+            'sys_info': sys_info,
             'v1': {
                 '_beta_keys': ['subplatform'],
                 'availability-zone': 'myaz',
                 'availability_zone': 'myaz',
                 'cloud-name': 'subclasscloudname',
                 'cloud_name': 'subclasscloudname',
+                'distro': 'ubuntu',
+                'distro_release': 'focal',
+                'distro_version': '20.04',
                 'instance-id': 'iid-datasource',
                 'instance_id': 'iid-datasource',
                 'local-hostname': 'test-subclass-hostname',
                 'local_hostname': 'test-subclass-hostname',
+                'kernel_release': '5.4.0-24-generic',
+                'machine': 'x86_64',
                 'platform': 'mytestsubclass',
                 'public_ssh_keys': [],
+                'python_version': '3.7',
                 'region': 'myregion',
-                'subplatform': 'unknown'},
+                'system_platform':
+                    'Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal',
+                'subplatform': 'unknown',
+                'variant': 'ubuntu'},
             'ds': {
                 '_doc': EXPERIMENTAL_TEXT,
                 'meta_data': {
@@ -359,7 +401,7 @@ class TestDataSource(CiTestCase):
                     'region': 'myregion',
                     'some': {'security-credentials': REDACT_SENSITIVE_VALUE}}}
         }
-        self.assertEqual(expected, redacted)
+        self.assertItemsEqual(expected, redacted)
         file_stat = os.stat(json_file)
         self.assertEqual(0o644, stat.S_IMODE(file_stat.st_mode))
 
@@ -376,28 +418,54 @@ class TestDataSource(CiTestCase):
                 'region': 'myregion',
                 'some': {'security-credentials': {
                     'cred1': 'sekret', 'cred2': 'othersekret'}}})
-        self.assertEqual(
-            ('security-credentials',), datasource.sensitive_metadata_keys)
-        datasource.get_data()
+        sys_info = {
+            "python": "3.7",
+            "platform":
+                "Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal",
+            "uname": ["Linux", "myhost", "5.4.0-24-generic", "SMP blah",
+                      "x86_64"],
+            "variant": "ubuntu", "dist": ["ubuntu", "20.04", "focal"]}
+
+        self.assertItemsEqual(
+            ('merged_cfg', 'security-credentials',),
+            datasource.sensitive_metadata_keys)
+        with mock.patch("cloudinit.util.system_info", return_value=sys_info):
+            datasource.get_data()
         sensitive_json_file = self.tmp_path(INSTANCE_JSON_SENSITIVE_FILE, tmp)
         content = util.load_file(sensitive_json_file)
         expected = {
             'base64_encoded_keys': [],
-            'sensitive_keys': ['ds/meta_data/some/security-credentials'],
+            'merged_cfg': {
+                 '_doc': (
+                     'Merged cloud-init system config from '
+                     '/etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/'),
+                 'datasource': {'_undef': {'key1': False}}},
+            'sensitive_keys': [
+                'ds/meta_data/some/security-credentials', 'merged_cfg'],
+            'sys_info': sys_info,
             'v1': {
                 '_beta_keys': ['subplatform'],
                 'availability-zone': 'myaz',
                 'availability_zone': 'myaz',
                 'cloud-name': 'subclasscloudname',
                 'cloud_name': 'subclasscloudname',
+                'distro': 'ubuntu',
+                'distro_release': 'focal',
+                'distro_version': '20.04',
                 'instance-id': 'iid-datasource',
                 'instance_id': 'iid-datasource',
+                'kernel_release': '5.4.0-24-generic',
                 'local-hostname': 'test-subclass-hostname',
                 'local_hostname': 'test-subclass-hostname',
+                'machine': 'x86_64',
                 'platform': 'mytestsubclass',
                 'public_ssh_keys': [],
+                'python_version': '3.7',
                 'region': 'myregion',
-                'subplatform': 'unknown'},
+                'subplatform': 'unknown',
+                'system_platform':
+                    'Linux-5.4.0-24-generic-x86_64-with-Ubuntu-20.04-focal',
+                'variant': 'ubuntu'},
             'ds': {
                 '_doc': EXPERIMENTAL_TEXT,
                 'meta_data': {
@@ -408,7 +476,7 @@ class TestDataSource(CiTestCase):
                         'security-credentials':
                             {'cred1': 'sekret', 'cred2': 'othersekret'}}}}
         }
-        self.assertEqual(expected, util.load_json(content))
+        self.assertItemsEqual(expected, util.load_json(content))
         file_stat = os.stat(sensitive_json_file)
         self.assertEqual(0o600, stat.S_IMODE(file_stat.st_mode))
         self.assertEqual(expected, util.load_json(content))
diff --git a/cloudinit/util.py b/cloudinit/util.py
index c02b3d9a..132f6051 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -656,7 +656,7 @@ def system_info():
         'system': platform.system(),
         'release': platform.release(),
         'python': platform.python_version(),
-        'uname': platform.uname(),
+        'uname': list(platform.uname()),
         'dist': get_linux_distro()
     }
     system = info['system'].lower()
diff --git a/doc/rtd/topics/instancedata.rst b/doc/rtd/topics/instancedata.rst
index 4227c4fd..845098bb 100644
--- a/doc/rtd/topics/instancedata.rst
+++ b/doc/rtd/topics/instancedata.rst
@@ -76,6 +76,11 @@ There are three basic top-level keys:
   'security sensitive'. Only the keys listed here will be redacted from
   instance-data.json for non-root users.
 
+* **merged_cfg**: Merged cloud-init 'system_config' from `/etc/cloud/cloud.cfg`
+  and  `/etc/cloud/cloud-cfg.d`. Values under this key could contain sensitive
+  information such as passwords, so it is included in the **sensitive-keys**
+  list which is only readable by root.
+
 * **ds**: Datasource-specific metadata crawled for the specific cloud
   platform. It should closely represent the structure of the cloud metadata
   crawled. The structure of content and details provided are entirely
@@ -83,6 +88,9 @@ There are three basic top-level keys:
   The content exposed under the 'ds' key is currently **experimental** and
   expected to change slightly in the upcoming cloud-init release.
 
+* **sys_info**: Information about the underlying os, python, architecture and
+  kernel. This represents the data collected by `cloudinit.util.system_info`.
+
 * **v1**: Standardized cloud-init metadata keys, these keys are guaranteed to
   exist on all cloud platforms. They will also retain their current behavior
   and format and will be carried forward even if cloud-init introduces a new
@@ -117,6 +125,21 @@ Example output:
 - nocloud
 - ovf
 
+v1.distro, v1.distro_version, v1.distro_release
+-----------------------------------------------
+This shall be the distro name, version and release as determined by
+`cloudinit.util.get_linux_distro`.
+
+Example output:
+
+- centos, 7.5, core
+- debian, 9, stretch
+- freebsd, 12.0-release-p10,
+- opensuse, 42.3, x86_64
+- opensuse-tumbleweed, 20180920, x86_64
+- redhat, 7.5, 'maipo'
+- sles, 12.3, x86_64
+- ubuntu, 20.04, focal
 
 v1.instance_id
 --------------
@@ -126,6 +149,14 @@ Examples output:
 
 - i-<hash>
 
+v1.kernel_release
+-----------------
+This shall be the running kernel `uname -r`
+
+Example output:
+
+- 5.3.0-1010-aws
+
 v1.local_hostname
 -----------------
 The internal or local hostname of the system.
@@ -135,6 +166,17 @@ Examples output:
 - ip-10-41-41-70
 - <user-provided-hostname>
 
+v1.machine
+----------
+This shall be the running cpu machine architecture `uname -m`
+
+Example output:
+
+- x86_64
+- i686
+- ppc64le
+- s390x
+
 v1.platform
 -------------
 An attempt to identify the cloud platfrom instance that the system is running
@@ -154,7 +196,7 @@ v1.subplatform
 Additional platform details describing the specific source or type of metadata
 used. The format of subplatform will be:
 
-``<subplatform_type> (<url_file_or_dev_path>``
+``<subplatform_type> (<url_file_or_dev_path>)``
 
 Examples output:
 
@@ -171,6 +213,15 @@ Examples output:
 
 - ['ssh-rsa AA...', ...]
 
+v1.python_version
+-----------------
+The version of python that is running cloud-init as determined by
+`cloudinit.util.system_info`
+
+Example output:
+
+- 3.7.6
+
 v1.region
 ---------
 The physical region/data center in which the instance is deployed.
@@ -192,164 +243,265 @@ Examples output:
 Example Output
 --------------
 
-Below is an example of ``/run/cloud-init/instance_data.json`` on an EC2
-instance:
+Below is an example of ``/run/cloud-init/instance-data-sensitive.json`` on an
+EC2 instance:
 
 .. sourcecode:: json
 
   {
+   "_beta_keys": [
+    "subplatform"
+   ],
+   "availability_zone": "us-east-1b",
    "base64_encoded_keys": [],
+   "merged_cfg": {
+    "_doc": "Merged cloud-init system config from /etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/",
+    "_log": [
+     "[loggers]\nkeys=root,cloudinit\n\n[handlers]\nkeys=consoleHandler,cloudLogHandler\n\n[formatters]\nkeys=simpleFormatter,arg0Formatter\n\n[logger_root]\nlevel=DEBUG\nhandlers=consoleHandler,cloudLogHandler\n\n[logger_cloudinit]\nlevel=DEBUG\nqualname=cloudinit\nhandlers=\npropagate=1\n\n[handler_consoleHandler]\nclass=StreamHandler\nlevel=WARNING\nformatter=arg0Formatter\nargs=(sys.stderr,)\n\n[formatter_arg0Formatter]\nformat=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s\n\n[formatter_simpleFormatter]\nformat=[CLOUDINIT] %(filename)s[%(levelname)s]: %(message)s\n",
+     "[handler_cloudLogHandler]\nclass=FileHandler\nlevel=DEBUG\nformatter=arg0Formatter\nargs=('/var/log/cloud-init.log',)\n",
+     "[handler_cloudLogHandler]\nclass=handlers.SysLogHandler\nlevel=DEBUG\nformatter=simpleFormatter\nargs=(\"/dev/log\", handlers.SysLogHandler.LOG_USER)\n"
+    ],
+    "cloud_config_modules": [
+     "emit_upstart",
+     "snap",
+     "ssh-import-id",
+     "locale",
+     "set-passwords",
+     "grub-dpkg",
+     "apt-pipelining",
+     "apt-configure",
+     "ubuntu-advantage",
+     "ntp",
+     "timezone",
+     "disable-ec2-metadata",
+     "runcmd",
+     "byobu"
+    ],
+    "cloud_final_modules": [
+     "package-update-upgrade-install",
+     "fan",
+     "landscape",
+     "lxd",
+     "ubuntu-drivers",
+     "puppet",
+     "chef",
+     "mcollective",
+     "salt-minion",
+     "rightscale_userdata",
+     "scripts-vendor",
+     "scripts-per-once",
+     "scripts-per-boot",
+     "scripts-per-instance",
+     "scripts-user",
+     "ssh-authkey-fingerprints",
+     "keys-to-console",
+     "phone-home",
+     "final-message",
+     "power-state-change"
+    ],
+    "cloud_init_modules": [
+     "migrator",
+     "seed_random",
+     "bootcmd",
+     "write-files",
+     "growpart",
+     "resizefs",
+     "disk_setup",
+     "mounts",
+     "set_hostname",
+     "update_hostname",
+     "update_etc_hosts",
+     "ca-certs",
+     "rsyslog",
+     "users-groups",
+     "ssh"
+    ],
+    "datasource_list": [
+     "Ec2",
+     "None"
+    ],
+    "def_log_file": "/var/log/cloud-init.log",
+    "disable_root": true,
+    "log_cfgs": [
+     [
+      "[loggers]\nkeys=root,cloudinit\n\n[handlers]\nkeys=consoleHandler,cloudLogHandler\n\n[formatters]\nkeys=simpleFormatter,arg0Formatter\n\n[logger_root]\nlevel=DEBUG\nhandlers=consoleHandler,cloudLogHandler\n\n[logger_cloudinit]\nlevel=DEBUG\nqualname=cloudinit\nhandlers=\npropagate=1\n\n[handler_consoleHandler]\nclass=StreamHandler\nlevel=WARNING\nformatter=arg0Formatter\nargs=(sys.stderr,)\n\n[formatter_arg0Formatter]\nformat=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s\n\n[formatter_simpleFormatter]\nformat=[CLOUDINIT] %(filename)s[%(levelname)s]: %(message)s\n",
+      "[handler_cloudLogHandler]\nclass=FileHandler\nlevel=DEBUG\nformatter=arg0Formatter\nargs=('/var/log/cloud-init.log',)\n"
+     ]
+    ],
+    "output": {
+     "all": "| tee -a /var/log/cloud-init-output.log"
+    },
+    "preserve_hostname": false,
+    "syslog_fix_perms": [
+     "syslog:adm",
+     "root:adm",
+     "root:wheel",
+     "root:root"
+    ],
+    "users": [
+     "default"
+    ],
+    "vendor_data": {
+     "enabled": true,
+     "prefix": []
+    }
+   },
+   "cloud_name": "aws",
+   "distro": "ubuntu",
+   "distro_release": "focal",
+   "distro_version": "20.04",
    "ds": {
     "_doc": "EXPERIMENTAL: The structure and format of content scoped under the 'ds' key may change in subsequent releases of cloud-init.",
     "_metadata_api_version": "2016-09-02",
     "dynamic": {
-     "instance-identity": {
+     "instance_identity": {
       "document": {
-       "accountId": "437526006925",
+       "accountId": "329910648901",
        "architecture": "x86_64",
-       "availabilityZone": "us-east-2b",
+       "availabilityZone": "us-east-1b",
        "billingProducts": null,
        "devpayProductCodes": null,
-       "imageId": "ami-079638aae7046bdd2",
-       "instanceId": "i-075f088c72ad3271c",
+       "imageId": "ami-02e8aa396f8be3b6d",
+       "instanceId": "i-0929128ff2f73a2f1",
        "instanceType": "t2.micro",
        "kernelId": null,
        "marketplaceProductCodes": null,
-       "pendingTime": "2018-10-05T20:10:43Z",
-       "privateIp": "10.41.41.95",
+       "pendingTime": "2020-02-27T20:46:18Z",
+       "privateIp": "172.31.81.43",
        "ramdiskId": null,
-       "region": "us-east-2",
+       "region": "us-east-1",
        "version": "2017-09-30"
       },
       "pkcs7": [
-       "MIAGCSqGSIb3DQEHAqCAMIACAQExCzAJBgUrDgMCGgUAMIAGCSqGSIb3DQEHAaCAJIAEggHbewog",
-       "ICJkZXZwYXlQcm9kdWN0Q29kZXMiIDogbnVsbCwKICAibWFya2V0cGxhY2VQcm9kdWN0Q29kZXMi",
-       "IDogbnVsbCwKICAicHJpdmF0ZUlwIiA6ICIxMC40MS40MS45NSIsCiAgInZlcnNpb24iIDogIjIw",
-       "MTctMDktMzAiLAogICJpbnN0YW5jZUlkIiA6ICJpLTA3NWYwODhjNzJhZDMyNzFjIiwKICAiYmls",
-       "bGluZ1Byb2R1Y3RzIiA6IG51bGwsCiAgImluc3RhbmNlVHlwZSIgOiAidDIubWljcm8iLAogICJh",
-       "Y2NvdW50SWQiIDogIjQzNzUyNjAwNjkyNSIsCiAgImF2YWlsYWJpbGl0eVpvbmUiIDogInVzLWVh",
-       "c3QtMmIiLAogICJrZXJuZWxJZCIgOiBudWxsLAogICJyYW1kaXNrSWQiIDogbnVsbCwKICAiYXJj",
-       "aGl0ZWN0dXJlIiA6ICJ4ODZfNjQiLAogICJpbWFnZUlkIiA6ICJhbWktMDc5NjM4YWFlNzA0NmJk",
-       "ZDIiLAogICJwZW5kaW5nVGltZSIgOiAiMjAxOC0xMC0wNVQyMDoxMDo0M1oiLAogICJyZWdpb24i",
-       "IDogInVzLWVhc3QtMiIKfQAAAAAAADGCARcwggETAgEBMGkwXDELMAkGA1UEBhMCVVMxGTAXBgNV",
-       "BAgTEFdhc2hpbmd0b24gU3RhdGUxEDAOBgNVBAcTB1NlYXR0bGUxIDAeBgNVBAoTF0FtYXpvbiBX",
-       "ZWIgU2VydmljZXMgTExDAgkAlrpI2eVeGmcwCQYFKw4DAhoFAKBdMBgGCSqGSIb3DQEJAzELBgkq",
-       "hkiG9w0BBwEwHAYJKoZIhvcNAQkFMQ8XDTE4MTAwNTIwMTA0OFowIwYJKoZIhvcNAQkEMRYEFK0k",
-       "Tz6n1A8/zU1AzFj0riNQORw2MAkGByqGSM44BAMELjAsAhRNrr174y98grPBVXUforN/6wZp8AIU",
-       "JLZBkrB2GJA8A4WJ1okq++jSrBIAAAAAAAA="
+       "MIAGCSqGSIb3DQ...",
+       "REDACTED",
+       "AhQUgq0iPWqPTVnT96tZE6L1XjjLHQAAAAAAAA=="
       ],
       "rsa2048": [
-       "MIAGCSqGSIb3DQEHAqCAMIACAQExDzANBglghkgBZQMEAgEFADCABgkqhkiG9w0BBwGggCSABIIB",
-       "23sKICAiZGV2cGF5UHJvZHVjdENvZGVzIiA6IG51bGwsCiAgIm1hcmtldHBsYWNlUHJvZHVjdENv",
-       "ZGVzIiA6IG51bGwsCiAgInByaXZhdGVJcCIgOiAiMTAuNDEuNDEuOTUiLAogICJ2ZXJzaW9uIiA6",
-       "ICIyMDE3LTA5LTMwIiwKICAiaW5zdGFuY2VJZCIgOiAiaS0wNzVmMDg4YzcyYWQzMjcxYyIsCiAg",
-       "ImJpbGxpbmdQcm9kdWN0cyIgOiBudWxsLAogICJpbnN0YW5jZVR5cGUiIDogInQyLm1pY3JvIiwK",
-       "ICAiYWNjb3VudElkIiA6ICI0Mzc1MjYwMDY5MjUiLAogICJhdmFpbGFiaWxpdHlab25lIiA6ICJ1",
-       "cy1lYXN0LTJiIiwKICAia2VybmVsSWQiIDogbnVsbCwKICAicmFtZGlza0lkIiA6IG51bGwsCiAg",
-       "ImFyY2hpdGVjdHVyZSIgOiAieDg2XzY0IiwKICAiaW1hZ2VJZCIgOiAiYW1pLTA3OTYzOGFhZTcw",
-       "NDZiZGQyIiwKICAicGVuZGluZ1RpbWUiIDogIjIwMTgtMTAtMDVUMjA6MTA6NDNaIiwKICAicmVn",
-       "aW9uIiA6ICJ1cy1lYXN0LTIiCn0AAAAAAAAxggH/MIIB+wIBATBpMFwxCzAJBgNVBAYTAlVTMRkw",
-       "FwYDVQQIExBXYXNoaW5ndG9uIFN0YXRlMRAwDgYDVQQHEwdTZWF0dGxlMSAwHgYDVQQKExdBbWF6",
-       "b24gV2ViIFNlcnZpY2VzIExMQwIJAM07oeX4xevdMA0GCWCGSAFlAwQCAQUAoGkwGAYJKoZIhvcN",
-       "AQkDMQsGCSqGSIb3DQEHATAcBgkqhkiG9w0BCQUxDxcNMTgxMDA1MjAxMDQ4WjAvBgkqhkiG9w0B",
-       "CQQxIgQgkYz0pZk3zJKBi4KP4egeOKJl/UYwu5UdE7id74pmPwMwDQYJKoZIhvcNAQEBBQAEggEA",
-       "dC3uIGGNul1OC1mJKSH3XoBWsYH20J/xhIdftYBoXHGf2BSFsrs9ZscXd2rKAKea4pSPOZEYMXgz",
-       "lPuT7W0WU89N3ZKviy/ReMSRjmI/jJmsY1lea6mlgcsJXreBXFMYucZvyeWGHdnCjamoKWXkmZlM",
-       "mSB1gshWy8Y7DzoKviYPQZi5aI54XK2Upt4kGme1tH1NI2Cq+hM4K+adxTbNhS3uzvWaWzMklUuU",
-       "QHX2GMmjAVRVc8vnA8IAsBCJJp+gFgYzi09IK+cwNgCFFPADoG6jbMHHf4sLB3MUGpiA+G9JlCnM",
-       "fmkjI2pNRB8spc0k4UG4egqLrqCz67WuK38tjwAAAAAAAA=="
+       "MIAGCSqGSIb...",
+       "REDACTED",
+       "clYQvuE45xXm7Yreg3QtQbrP//owl1eZHj6s350AAAAAAAA="
       ],
       "signature": [
-       "Tsw6h+V3WnxrNVSXBYIOs1V4j95YR1mLPPH45XnhX0/Ei3waJqf7/7EEKGYP1Cr4PTYEULtZ7Mvf",
-       "+xJpM50Ivs2bdF7o0c4vnplRWe3f06NI9pv50dr110j/wNzP4MZ1pLhJCqubQOaaBTF3LFutgRrt",
-       "r4B0mN3p7EcqD8G+ll0="
+       "dA+QV+LLCWCRNddnrKleYmh2GvYo+t8urDkdgmDSsPi",
+       "REDACTED",
+       "kDT4ygyJLFkd3b4qjAs="
       ]
      }
     },
-    "meta-data": {
-     "ami-id": "ami-079638aae7046bdd2",
-     "ami-launch-index": "0",
-     "ami-manifest-path": "(unknown)",
-     "block-device-mapping": {
+    "meta_data": {
+     "ami_id": "ami-02e8aa396f8be3b6d",
+     "ami_launch_index": "0",
+     "ami_manifest_path": "(unknown)",
+     "block_device_mapping": {
       "ami": "/dev/sda1",
-      "ephemeral0": "sdb",
-      "ephemeral1": "sdc",
       "root": "/dev/sda1"
      },
-     "hostname": "ip-10-41-41-95.us-east-2.compute.internal",
-     "instance-action": "none",
-     "instance-id": "i-075f088c72ad3271c",
-     "instance-type": "t2.micro",
-     "local-hostname": "ip-10-41-41-95.us-east-2.compute.internal",
-     "local-ipv4": "10.41.41.95",
-     "mac": "06:74:8f:39:cd:a6",
+     "hostname": "ip-172-31-81-43.ec2.internal",
+     "instance_action": "none",
+     "instance_id": "i-0929128ff2f73a2f1",
+     "instance_type": "t2.micro",
+     "local_hostname": "ip-172-31-81-43.ec2.internal",
+     "local_ipv4": "172.31.81.43",
+     "mac": "12:7e:c9:93:29:af",
      "metrics": {
       "vhostmd": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
      },
      "network": {
       "interfaces": {
        "macs": {
-       "06:74:8f:39:cd:a6": {
-        "device-number": "0",
-        "interface-id": "eni-052058bbd7831eaae",
-        "ipv4-associations": {
-         "18.218.221.122": "10.41.41.95"
-        },
-        "local-hostname": "ip-10-41-41-95.us-east-2.compute.internal",
-        "local-ipv4s": "10.41.41.95",
-        "mac": "06:74:8f:39:cd:a6",
-        "owner-id": "437526006925",
-        "public-hostname": "ec2-18-218-221-122.us-east-2.compute.amazonaws.com",
-        "public-ipv4s": "18.218.221.122",
-        "security-group-ids": "sg-828247e9",
-        "security-groups": "Cloud-init integration test secgroup",
-        "subnet-id": "subnet-282f3053",
-        "subnet-ipv4-cidr-block": "10.41.41.0/24",
-        "subnet-ipv6-cidr-blocks": "2600:1f16:b80:ad00::/64",
-        "vpc-id": "vpc-252ef24d",
-        "vpc-ipv4-cidr-block": "10.41.0.0/16",
-        "vpc-ipv4-cidr-blocks": "10.41.0.0/16",
-        "vpc-ipv6-cidr-blocks": "2600:1f16:b80:ad00::/56"
-       }
+        "12:7e:c9:93:29:af": {
+         "device_number": "0",
+         "interface_id": "eni-0c07a0474339b801d",
+         "ipv4_associations": {
+          "3.89.187.177": "172.31.81.43"
+         },
+         "local_hostname": "ip-172-31-81-43.ec2.internal",
+         "local_ipv4s": "172.31.81.43",
+         "mac": "12:7e:c9:93:29:af",
+         "owner_id": "329910648901",
+         "public_hostname": "ec2-3-89-187-177.compute-1.amazonaws.com",
+         "public_ipv4s": "3.89.187.177",
+         "security_group_ids": "sg-0100038b68aa79986",
+         "security_groups": "launch-wizard-3",
+         "subnet_id": "subnet-04e2d12a",
+         "subnet_ipv4_cidr_block": "172.31.80.0/20",
+         "vpc_id": "vpc-210b4b5b",
+         "vpc_ipv4_cidr_block": "172.31.0.0/16",
+         "vpc_ipv4_cidr_blocks": "172.31.0.0/16"
+        }
        }
       }
      },
      "placement": {
-      "availability-zone": "us-east-2b"
+      "availability_zone": "us-east-1b"
      },
      "profile": "default-hvm",
-     "public-hostname": "ec2-18-218-221-122.us-east-2.compute.amazonaws.com",
-     "public-ipv4": "18.218.221.122",
-     "public-keys": {
-      "cloud-init-integration": [
-       "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSL7uWGj8cgWyIOaspgKdVy0cKJ+UTjfv7jBOjG2H/GN8bJVXy72XAvnhM0dUM+CCs8FOf0YlPX+Frvz2hKInrmRhZVwRSL129PasD12MlI3l44u6IwS1o/W86Q+tkQYEljtqDOo0a+cOsaZkvUNzUyEXUwz/lmYa6G4hMKZH4NBj7nbAAF96wsMCoyNwbWryBnDYUr6wMbjRR1J9Pw7Xh7WRC73wy4Va2YuOgbD3V/5ZrFPLbWZW/7TFXVrql04QVbyei4aiFR5n//GvoqwQDNe58LmbzX/xvxyKJYdny2zXmdAhMxbrpFQsfpkJ9E/H5w0yOdSvnWbUoG5xNGoOB cloud-init-integration"
-      ]
-     },
-     "reservation-id": "r-0594a20e31f6cfe46",
-     "security-groups": "Cloud-init integration test secgroup",
+     "public_hostname": "ec2-3-89-187-177.compute-1.amazonaws.com",
+     "public_ipv4": "3.89.187.177",
+     "reservation_id": "r-0c481643d15766a02",
+     "security_groups": "launch-wizard-3",
      "services": {
       "domain": "amazonaws.com",
       "partition": "aws"
      }
     }
    },
+   "instance_id": "i-0929128ff2f73a2f1",
+   "kernel_release": "5.3.0-1010-aws",
+   "local_hostname": "ip-172-31-81-43",
+   "machine": "x86_64",
+   "platform": "ec2",
+   "public_ssh_keys": [],
+   "python_version": "3.7.6",
+   "region": "us-east-1",
    "sensitive_keys": [],
+   "subplatform": "metadata (http://169.254.169.254)",
+   "sys_info": {
+    "dist": [
+     "ubuntu",
+     "20.04",
+     "focal"
+    ],
+    "platform": "Linux-5.3.0-1010-aws-x86_64-with-Ubuntu-20.04-focal",
+    "python": "3.7.6",
+    "release": "5.3.0-1010-aws",
+    "system": "Linux",
+    "uname": [
+     "Linux",
+     "ip-172-31-81-43",
+     "5.3.0-1010-aws",
+     "#11-Ubuntu SMP Thu Jan 16 07:59:32 UTC 2020",
+     "x86_64",
+     "x86_64"
+    ],
+    "variant": "ubuntu"
+   },
+   "system_platform": "Linux-5.3.0-1010-aws-x86_64-with-Ubuntu-20.04-focal",
+   "userdata": "#cloud-config\nssh_import_id: [<my-launchpad-id>]\n...",
    "v1": {
     "_beta_keys": [
      "subplatform"
     ],
-    "availability-zone": "us-east-2b",
-    "availability_zone": "us-east-2b",
+    "availability_zone": "us-east-1b",
     "cloud_name": "aws",
-    "instance_id": "i-075f088c72ad3271c",
-    "local_hostname": "ip-10-41-41-95",
+    "distro": "ubuntu",
+    "distro_release": "focal",
+    "distro_version": "20.04",
+    "instance_id": "i-0929128ff2f73a2f1",
+    "kernel": "5.3.0-1010-aws",
+    "local_hostname": "ip-172-31-81-43",
+    "machine": "x86_64",
     "platform": "ec2",
-    "public_ssh_keys": [
-     "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSL7uWGj8cgWyIOaspgKdVy0cKJ+UTjfv7jBOjG2H/GN8bJVXy72XAvnhM0dUM+CCs8FOf0YlPX+Frvz2hKInrmRhZVwRSL129PasD12MlI3l44u6IwS1o/W86Q+tkQYEljtqDOo0a+cOsaZkvUNzUyEXUwz/lmYa6G4hMKZH4NBj7nbAAF96wsMCoyNwbWryBnDYUr6wMbjRR1J9Pw7Xh7WRC73wy4Va2YuOgbD3V/5ZrFPLbWZW/7TFXVrql04QVbyei4aiFR5n//GvoqwQDNe58LmbzX/xvxyKJYdny2zXmdAhMxbrpFQsfpkJ9E/H5w0yOdSvnWbUoG5xNGoOB cloud-init-integration"
-    ],
-    "region": "us-east-2",
-    "subplatform": "metadata (http://169.254.169.254)"
-   }
+    "public_ssh_keys": [],
+    "python": "3.7.6",
+    "region": "us-east-1",
+    "subplatform": "metadata (http://169.254.169.254)",
+    "system_platform": "Linux-5.3.0-1010-aws-x86_64-with-Ubuntu-20.04-focal",
+    "variant": "ubuntu"
+   },
+   "variant": "ubuntu",
+   "vendordata": ""
   }
 
 
diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index fd12d87b..5976e234 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -172,9 +172,7 @@ class CloudTestCase(unittest2.TestCase):
                 'Skipping instance-data.json test.'
                 ' OS: %s not bionic or newer' % self.os_name)
         instance_data = json.loads(out)
-        self.assertItemsEqual(
-            [],
-            instance_data['base64_encoded_keys'])
+        self.assertItemsEqual(['ci_cfg'], instance_data['sensitive_keys'])
         ds = instance_data.get('ds', {})
         v1_data = instance_data.get('v1', {})
         metadata = ds.get('meta-data', {})
@@ -201,6 +199,23 @@ class CloudTestCase(unittest2.TestCase):
         self.assertIn('i-', v1_data['instance_id'])
         self.assertIn('ip-', v1_data['local_hostname'])
         self.assertIsNotNone(v1_data['region'], 'expected ec2 region')
+        self.assertIsNotNone(
+            re.match(r'\d\.\d+\.\d+-\d+-aws', v1_data['kernel_release']))
+        self.assertEqual(
+            'redacted for non-root user', instance_data['merged_cfg'])
+        self.assertEqual(self.os_cfg['os'], v1_data['variant'])
+        self.assertEqual(self.os_cfg['os'], v1_data['distro'])
+        self.assertEqual(
+            self.os_cfg['os'], instance_data["sys_info"]['dist'][0],
+            "Unexpected sys_info dist value")
+        self.assertEqual(self.os_name, v1_data['distro_release'])
+        self.assertEqual(
+            str(self.os_cfg['version']), v1_data['distro_version'])
+        self.assertEqual('x86_64', v1_data['machine'])
+        self.assertIsNotNone(
+            re.match(r'3.\d\.\d', v1_data['python_version']),
+            "unexpected python version: {ver}".format(
+                ver=v1_data["python_version"]))
 
     def test_instance_data_json_lxd(self):
         """Validate instance-data.json content by lxd platform.
@@ -237,6 +252,23 @@ class CloudTestCase(unittest2.TestCase):
         self.assertIsNone(
             v1_data['region'],
             'found unexpected lxd region %s' % v1_data['region'])
+        self.assertIsNotNone(
+            re.match(r'\d\.\d+\.\d+-\d+', v1_data['kernel_release']))
+        self.assertEqual(
+            'redacted for non-root user', instance_data['merged_cfg'])
+        self.assertEqual(self.os_cfg['os'], v1_data['variant'])
+        self.assertEqual(self.os_cfg['os'], v1_data['distro'])
+        self.assertEqual(
+            self.os_cfg['os'], instance_data["sys_info"]['dist'][0],
+            "Unexpected sys_info dist value")
+        self.assertEqual(self.os_name, v1_data['distro_release'])
+        self.assertEqual(
+            str(self.os_cfg['version']), v1_data['distro_version'])
+        self.assertEqual('x86_64', v1_data['machine'])
+        self.assertIsNotNone(
+            re.match(r'3.\d\.\d', v1_data['python_version']),
+            "unexpected python version: {ver}".format(
+                ver=v1_data["python_version"]))
 
     def test_instance_data_json_kvm(self):
         """Validate instance-data.json content by nocloud-kvm platform.
@@ -278,6 +310,23 @@ class CloudTestCase(unittest2.TestCase):
         self.assertIsNone(
             v1_data['region'],
             'found unexpected lxd region %s' % v1_data['region'])
+        self.assertIsNotNone(
+            re.match(r'\d\.\d+\.\d+-\d+', v1_data['kernel_release']))
+        self.assertEqual(
+            'redacted for non-root user', instance_data['merged_cfg'])
+        self.assertEqual(self.os_cfg['os'], v1_data['variant'])
+        self.assertEqual(self.os_cfg['os'], v1_data['distro'])
+        self.assertEqual(
+            self.os_cfg['os'], instance_data["sys_info"]['dist'][0],
+            "Unexpected sys_info dist value")
+        self.assertEqual(self.os_name, v1_data['distro_release'])
+        self.assertEqual(
+                str(self.os_cfg['version']), v1_data['distro_version'])
+        self.assertEqual('x86_64', v1_data['machine'])
+        self.assertIsNotNone(
+            re.match(r'3.\d\.\d', v1_data['python_version']),
+            "unexpected python version: {ver}".format(
+                ver=v1_data["python_version"]))
 
 
 class PasswordListTest(CloudTestCase):
diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py
index d62d542b..7aa3b1d1 100644
--- a/tests/unittests/test_datasource/test_cloudsigma.py
+++ b/tests/unittests/test_datasource/test_cloudsigma.py
@@ -3,6 +3,7 @@
 import copy
 
 from cloudinit.cs_utils import Cepko
+from cloudinit import distros
 from cloudinit import helpers
 from cloudinit import sources
 from cloudinit.sources import DataSourceCloudSigma
@@ -47,8 +48,11 @@ class DataSourceCloudSigmaTest(test_helpers.CiTestCase):
         self.paths = helpers.Paths({'run_dir': self.tmp_dir()})
         self.add_patch(DS_PATH + '.is_running_in_cloudsigma',
                        "m_is_container", return_value=True)
+
+        distro_cls = distros.fetch("ubuntu")
+        distro = distro_cls("ubuntu", cfg={}, paths=self.paths)
         self.datasource = DataSourceCloudSigma.DataSourceCloudSigma(
-            "", "", paths=self.paths)
+            sys_cfg={}, distro=distro, paths=self.paths)
         self.datasource.cepko = CepkoMock(SERVER_CONTEXT)
 
     def test_get_hostname(self):