T2117: Cloud-init updated to 20.3

Merged with 20.3 tag from the upstream Cloud-init repository

22 files changed, 284 insertions, 131 deletions

diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py
index f04d0cd4..e45ad947 100644
--- a/tests/cloud_tests/bddeb.py
+++ b/tests/cloud_tests/bddeb.py
@@ -6,7 +6,7 @@ from functools import partial
 import os
 import tempfile
 
-from cloudinit import util as c_util
+from cloudinit import subp
 from tests.cloud_tests import (config, LOG)
 from tests.cloud_tests import platforms
 from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single)
@@ -42,8 +42,8 @@ def build_deb(args, instance):
                'GIT_WORK_TREE': extract_dir}
 
     LOG.debug('creating tarball of cloud-init at: %s', local_tarball)
-    c_util.subp(['tar', 'cf', local_tarball, '--owner', 'root',
-                 '--group', 'root', '-C', args.cloud_init, '.'])
+    subp.subp(['tar', 'cf', local_tarball, '--owner', 'root',
+               '--group', 'root', '-C', args.cloud_init, '.'])
     LOG.debug('copying to remote system at: %s', remote_tarball)
     instance.push_file(local_tarball, remote_tarball)
 
@@ -55,7 +55,7 @@ def build_deb(args, instance):
     LOG.debug('installing deps')
     deps_path = os.path.join(extract_dir, 'tools', 'read-dependencies')
     instance.execute([deps_path, '--install', '--test-distro',
-                      '--distro', 'ubuntu', '--python-version', '3'])
+                      '--distro', 'ubuntu'])
 
     LOG.debug('building deb in remote system at: %s', output_link)
     bddeb_args = args.bddeb_args.split() if args.bddeb_args else []
diff --git a/tests/cloud_tests/platforms/__init__.py b/tests/cloud_tests/platforms/__init__.py
index 6a410b84..e506baa0 100644
--- a/tests/cloud_tests/platforms/__init__.py
+++ b/tests/cloud_tests/platforms/__init__.py
@@ -6,6 +6,7 @@ from .ec2 import platform as ec2
 from .lxd import platform as lxd
 from .nocloudkvm import platform as nocloudkvm
 from .azurecloud import platform as azurecloud
+from ..util import emit_dots_on_travis
 
 PLATFORMS = {
     'ec2': ec2.EC2Platform,
@@ -17,7 +18,8 @@ PLATFORMS = {
 
 def get_image(platform, config):
     """Get image from platform object using os_name."""
-    return platform.get_image(config)
+    with emit_dots_on_travis():
+        return platform.get_image(config)
 
 
 def get_instance(snapshot, *args, **kwargs):
diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py
index f1e28a96..eedbaae8 100644
--- a/tests/cloud_tests/platforms/azurecloud/instance.py
+++ b/tests/cloud_tests/platforms/azurecloud/instance.py
@@ -80,7 +80,6 @@ class AzureCloudInstance(Instance):
         except CloudError:
             LOG.debug(('image not found, launching instance with base image, '
                        'image_id=%s'), self.image_id)
-            pass
 
         vm_params = {
             'name': self.vm_name,
@@ -135,9 +134,10 @@ class AzureCloudInstance(Instance):
                                  self.vm_name, vm_params)
             LOG.debug('creating instance %s from image_id=%s', self.vm_name,
                       self.image_id)
-        except CloudError:
-            raise RuntimeError('failed creating instance:\n{}'.format(
-                traceback.format_exc()))
+        except CloudError as e:
+            raise RuntimeError(
+                'failed creating instance:\n{}'.format(traceback.format_exc())
+            ) from e
 
         if wait:
             self.instance.wait()
@@ -169,7 +169,6 @@ class AzureCloudInstance(Instance):
                     sleep(15)
                 else:
                     LOG.warning('Could not find console log: %s', e)
-                    pass
 
         LOG.debug('stopping instance %s', self.image_id)
         vm_deallocate = \
diff --git a/tests/cloud_tests/platforms/azurecloud/platform.py b/tests/cloud_tests/platforms/azurecloud/platform.py
index cb62a74b..a664f612 100644
--- a/tests/cloud_tests/platforms/azurecloud/platform.py
+++ b/tests/cloud_tests/platforms/azurecloud/platform.py
@@ -59,9 +59,12 @@ class AzureCloudPlatform(Platform):
             self.vnet = self._create_vnet()
             self.subnet = self._create_subnet()
             self.nic = self._create_nic()
-        except CloudError:
-            raise RuntimeError('failed creating a resource:\n{}'.format(
-                traceback.format_exc()))
+        except CloudError as e:
+            raise RuntimeError(
+                'failed creating a resource:\n{}'.format(
+                    traceback.format_exc()
+                )
+            ) from e
 
     def create_instance(self, properties, config, features,
                         image_id, user_data=None):
@@ -105,8 +108,10 @@ class AzureCloudPlatform(Platform):
             if image_id.find('__') > 0:
                 image_id = image_id.split('__')[1]
                 LOG.debug('image_id shortened to %s', image_id)
-        except KeyError:
-            raise RuntimeError('no images found for %s' % img_conf['release'])
+        except KeyError as e:
+            raise RuntimeError(
+                'no images found for %s' % img_conf['release']
+            ) from e
 
         return AzureCloudImage(self, img_conf, image_id)
 
@@ -140,9 +145,11 @@ class AzureCloudPlatform(Platform):
                 secret=azure_creds['clientSecret'],
                 tenant=azure_creds['tenantId'])
             return credentials, subscription_id
-        except KeyError:
-            raise RuntimeError('Please configure Azure service principal'
-                               ' credentials in %s' % cred_file)
+        except KeyError as e:
+            raise RuntimeError(
+                'Please configure Azure service principal'
+                ' credentials in %s' % cred_file
+            ) from e
 
     def _create_resource_group(self):
         """Create resource group"""
diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py
index ab6037b1..d2e84047 100644
--- a/tests/cloud_tests/platforms/ec2/instance.py
+++ b/tests/cloud_tests/platforms/ec2/instance.py
@@ -49,11 +49,11 @@ class EC2Instance(Instance):
             # OutputBytes comes from platform._decode_console_output_as_bytes
             response = self.instance.console_output()
             return response['OutputBytes']
-        except KeyError:
+        except KeyError as e:
             if 'Output' in response:
                 msg = ("'OutputBytes' did not exist in console_output() but "
                        "'Output' did: %s..." % response['Output'][0:128])
-                raise util.PlatformError('console_log', msg)
+                raise util.PlatformError('console_log', msg) from e
             return ('No Console Output [%s]' % self.instance).encode()
 
     def destroy(self):
diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py
index 7a3d0fe0..b61a2ffb 100644
--- a/tests/cloud_tests/platforms/ec2/platform.py
+++ b/tests/cloud_tests/platforms/ec2/platform.py
@@ -35,12 +35,14 @@ class EC2Platform(Platform):
             self.ec2_resource = b3session.resource('ec2')
             self.ec2_region = b3session.region_name
             self.key_name = self._upload_public_key(config)
-        except botocore.exceptions.NoRegionError:
+        except botocore.exceptions.NoRegionError as e:
             raise RuntimeError(
-                'Please configure default region in $HOME/.aws/config')
-        except botocore.exceptions.NoCredentialsError:
+                'Please configure default region in $HOME/.aws/config'
+            ) from e
+        except botocore.exceptions.NoCredentialsError as e:
             raise RuntimeError(
-                'Please configure ec2 credentials in $HOME/.aws/credentials')
+                'Please configure ec2 credentials in $HOME/.aws/credentials'
+            ) from e
 
         self.vpc = self._create_vpc()
         self.internet_gateway = self._create_internet_gateway()
@@ -125,8 +127,10 @@ class EC2Platform(Platform):
 
         try:
             image_ami = image['id']
-        except KeyError:
-            raise RuntimeError('No images found for %s!' % img_conf['release'])
+        except KeyError as e:
+            raise RuntimeError(
+                'No images found for %s!' % img_conf['release']
+            ) from e
 
         LOG.debug('found image: %s', image_ami)
         image = EC2Image(self, img_conf, image_ami)
@@ -195,7 +199,7 @@ class EC2Platform(Platform):
                 CidrBlock=self.ipv4_cidr,
                 AmazonProvidedIpv6CidrBlock=True)
         except botocore.exceptions.ClientError as e:
-            raise RuntimeError(e)
+            raise RuntimeError(e) from e
 
         vpc.wait_until_available()
         self._tag_resource(vpc)
diff --git a/tests/cloud_tests/platforms/images.py b/tests/cloud_tests/platforms/images.py
index 557a5cf6..f047de2e 100644
--- a/tests/cloud_tests/platforms/images.py
+++ b/tests/cloud_tests/platforms/images.py
@@ -52,6 +52,5 @@ class Image(TargetBase):
 
     def destroy(self):
         """Clean up data associated with image."""
-        pass
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/instances.py b/tests/cloud_tests/platforms/instances.py
index 529e79cd..efc35c7f 100644
--- a/tests/cloud_tests/platforms/instances.py
+++ b/tests/cloud_tests/platforms/instances.py
@@ -132,8 +132,8 @@ class Instance(TargetBase):
         """
         def clean_test(test):
             """Clean formatting for system ready test testcase."""
-            return ' '.join(l for l in test.strip().splitlines()
-                            if not l.lstrip().startswith('#'))
+            return ' '.join(line for line in test.strip().splitlines()
+                            if not line.lstrip().startswith('#'))
 
         boot_timeout = self.config['boot_timeout']
         tests = [self.config['system_ready_script']]
diff --git a/tests/cloud_tests/platforms/lxd/image.py b/tests/cloud_tests/platforms/lxd/image.py
index b5de1f52..a88b47f3 100644
--- a/tests/cloud_tests/platforms/lxd/image.py
+++ b/tests/cloud_tests/platforms/lxd/image.py
@@ -8,6 +8,7 @@ import tempfile
 
 from ..images import Image
 from .snapshot import LXDSnapshot
+from cloudinit import subp
 from cloudinit import util as c_util
 from tests.cloud_tests import util
 
@@ -75,19 +76,36 @@ class LXDImage(Image):
         }
 
     def export_image(self, output_dir):
-        """Export image from lxd image store to (split) tarball on disk.
+        """Export image from lxd image store to disk.
 
-        @param output_dir: dir to store tarballs in
-        @return_value: tuple of path to metadata tarball and rootfs tarball
+        @param output_dir: dir to store the exported image in
+        @return_value: tuple of path to metadata tarball and rootfs
+
+        Only the "split" image format with separate rootfs and metadata
+        files is supported, e.g:
+
+            71f171df[...]cd31.squashfs (could also be: .tar.xz or .tar.gz)
+            meta-71f171df[...]cd31.tar.xz
+
+        Combined images made by a single tarball are not supported.
         """
         # pylxd's image export feature doesn't do split exports, so use cmdline
-        c_util.subp(['lxc', 'image', 'export', self.pylxd_image.fingerprint,
-                     output_dir], capture=True)
-        tarballs = [p for p in os.listdir(output_dir) if p.endswith('tar.xz')]
+        fp = self.pylxd_image.fingerprint
+        subp.subp(['lxc', 'image', 'export', fp, output_dir], capture=True)
+        image_files = [p for p in os.listdir(output_dir) if fp in p]
+
+        if len(image_files) != 2:
+            raise NotImplementedError(
+                "Image %s has unsupported format. "
+                "Expected 2 files, found %d: %s."
+                % (fp, len(image_files), ', '.join(image_files)))
+
         metadata = os.path.join(
-            output_dir, next(p for p in tarballs if p.startswith('meta-')))
+            output_dir,
+            next(p for p in image_files if p.startswith('meta-')))
         rootfs = os.path.join(
-            output_dir, next(p for p in tarballs if not p.startswith('meta-')))
+            output_dir,
+            next(p for p in image_files if not p.startswith('meta-')))
         return (metadata, rootfs)
 
     def import_image(self, metadata, rootfs):
@@ -101,8 +119,8 @@ class LXDImage(Image):
         """
         alias = util.gen_instance_name(
             image_desc=str(self), use_desc='update-metadata')
-        c_util.subp(['lxc', 'image', 'import', metadata, rootfs,
-                     '--alias', alias], capture=True)
+        subp.subp(['lxc', 'image', 'import', metadata, rootfs,
+                   '--alias', alias], capture=True)
         self.pylxd_image = self.platform.query_image_by_alias(alias)
         return self.pylxd_image.fingerprint
 
diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py
index 2b804a62..2b973a08 100644
--- a/tests/cloud_tests/platforms/lxd/instance.py
+++ b/tests/cloud_tests/platforms/lxd/instance.py
@@ -7,7 +7,8 @@ import shutil
 import time
 from tempfile import mkdtemp
 
-from cloudinit.util import load_yaml, subp, ProcessExecutionError, which
+from cloudinit.subp import subp, ProcessExecutionError, which
+from cloudinit.util import load_yaml
 from tests.cloud_tests import LOG
 from tests.cloud_tests.util import PlatformError
 
@@ -174,7 +175,8 @@ class LXDInstance(Instance):
             raise PlatformError(
                 "console log",
                 "Console log failed [%d]: stdout=%s stderr=%s" % (
-                    e.exit_code, e.stdout, e.stderr))
+                    e.exit_code, e.stdout, e.stderr)
+            ) from e
 
     def reboot(self, wait=True):
         """Reboot instance."""
diff --git a/tests/cloud_tests/platforms/nocloudkvm/image.py b/tests/cloud_tests/platforms/nocloudkvm/image.py
index bc2b6e75..ff5b6ad7 100644
--- a/tests/cloud_tests/platforms/nocloudkvm/image.py
+++ b/tests/cloud_tests/platforms/nocloudkvm/image.py
@@ -2,7 +2,7 @@
 
 """NoCloud KVM Image Base Class."""
 
-from cloudinit import util as c_util
+from cloudinit import subp
 
 import os
 import shutil
@@ -30,8 +30,8 @@ class NoCloudKVMImage(Image):
         self._img_path = os.path.join(self._workd,
                                       os.path.basename(self._orig_img_path))
 
-        c_util.subp(['qemu-img', 'create', '-f', 'qcow2',
-                    '-b', orig_img_path, self._img_path])
+        subp.subp(['qemu-img', 'create', '-f', 'qcow2',
+                   '-b', orig_img_path, self._img_path])
 
         super(NoCloudKVMImage, self).__init__(platform, config)
 
@@ -50,10 +50,10 @@ class NoCloudKVMImage(Image):
                       '--system-resolvconf', self._img_path,
                       '--', 'chroot', '_MOUNTPOINT_']
         try:
-            out, err = c_util.subp(mic_chroot + env_args + list(command),
-                                   data=stdin, decode=False)
+            out, err = subp.subp(mic_chroot + env_args + list(command),
+                                 data=stdin, decode=False)
             return (out, err, 0)
-        except c_util.ProcessExecutionError as e:
+        except subp.ProcessExecutionError as e:
             return (e.stdout, e.stderr, e.exit_code)
 
     def snapshot(self):
diff --git a/tests/cloud_tests/platforms/nocloudkvm/instance.py b/tests/cloud_tests/platforms/nocloudkvm/instance.py
index 96185b75..5140a11c 100644
--- a/tests/cloud_tests/platforms/nocloudkvm/instance.py
+++ b/tests/cloud_tests/platforms/nocloudkvm/instance.py
@@ -11,7 +11,7 @@ import uuid
 
 from ..instances import Instance
 from cloudinit.atomic_helper import write_json
-from cloudinit import util as c_util
+from cloudinit import subp
 from tests.cloud_tests import LOG, util
 
 # This domain contains reverse lookups for hostnames that are used.
@@ -110,8 +110,8 @@ class NoCloudKVMInstance(Instance):
         """Clean up instance."""
         if self.pid:
             try:
-                c_util.subp(['kill', '-9', self.pid])
-            except c_util.ProcessExecutionError:
+                subp.subp(['kill', '-9', self.pid])
+            except subp.ProcessExecutionError:
                 pass
 
         if self.pid_file:
@@ -143,8 +143,8 @@ class NoCloudKVMInstance(Instance):
 
         # meta-data can be yaml, but more easily pretty printed with json
         write_json(meta_data_file, self.meta_data)
-        c_util.subp(['cloud-localds', seed_file, user_data_file,
-                     meta_data_file])
+        subp.subp(['cloud-localds', seed_file, user_data_file,
+                   meta_data_file])
 
         return seed_file
 
diff --git a/tests/cloud_tests/platforms/nocloudkvm/platform.py b/tests/cloud_tests/platforms/nocloudkvm/platform.py
index 2d1480f5..53c8ebf2 100644
--- a/tests/cloud_tests/platforms/nocloudkvm/platform.py
+++ b/tests/cloud_tests/platforms/nocloudkvm/platform.py
@@ -12,6 +12,7 @@ from simplestreams import util as s_util
 from ..platforms import Platform
 from .image import NoCloudKVMImage
 from .instance import NoCloudKVMInstance
+from cloudinit import subp
 from cloudinit import util as c_util
 from tests.cloud_tests import util
 
@@ -84,8 +85,8 @@ class NoCloudKVMPlatform(Platform):
         """
         name = util.gen_instance_name(image_desc=image_desc, use_desc=use_desc)
         img_path = os.path.join(self.config['data_dir'], name + '.qcow2')
-        c_util.subp(['qemu-img', 'create', '-f', 'qcow2',
-                    '-b', src_img_path, img_path])
+        subp.subp(['qemu-img', 'create', '-f', 'qcow2',
+                   '-b', src_img_path, img_path])
 
         return NoCloudKVMInstance(self, name, img_path, properties, config,
                                   features, user_data, meta_data)
diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py
index bebdf1c6..ac3b6563 100644
--- a/tests/cloud_tests/platforms/platforms.py
+++ b/tests/cloud_tests/platforms/platforms.py
@@ -7,6 +7,7 @@ import shutil
 from simplestreams import filters, mirrors
 from simplestreams import util as s_util
 
+from cloudinit import subp
 from cloudinit import util as c_util
 
 from tests.cloud_tests import util
@@ -48,10 +49,10 @@ class Platform(object):
         if os.path.exists(filename):
             c_util.del_file(filename)
 
-        c_util.subp(['ssh-keygen', '-m', 'PEM', '-t', 'rsa', '-b', '4096',
-                     '-f', filename, '-P', '',
-                     '-C', 'ubuntu@cloud_test'],
-                    capture=True)
+        subp.subp(['ssh-keygen', '-m', 'PEM', '-t', 'rsa', '-b', '4096',
+                   '-f', filename, '-P', '',
+                   '-C', 'ubuntu@cloud_test'],
+                  capture=True)
 
     @staticmethod
     def _query_streams(img_conf, img_filter):
@@ -73,8 +74,10 @@ class Platform(object):
 
         try:
             return tmirror.json_entries[0]
-        except IndexError:
-            raise RuntimeError('no images found with filter: %s' % img_filter)
+        except IndexError as e:
+            raise RuntimeError(
+                'no images found with filter: %s' % img_filter
+            ) from e
 
 
 class FilterMirror(mirrors.BasicMirrorWriter):
diff --git a/tests/cloud_tests/platforms/snapshots.py b/tests/cloud_tests/platforms/snapshots.py
index 94328982..0f5f8bb6 100644
--- a/tests/cloud_tests/platforms/snapshots.py
+++ b/tests/cloud_tests/platforms/snapshots.py
@@ -40,6 +40,5 @@ class Snapshot(object):
 
     def destroy(self):
         """Clean up snapshot data."""
-        pass
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml
index 7ddc5b85..e76a3d35 100644
--- a/tests/cloud_tests/releases.yaml
+++ b/tests/cloud_tests/releases.yaml
@@ -30,8 +30,10 @@ default_release_config:
         mirror_url: https://cloud-images.ubuntu.com/daily
         mirror_dir: '/srv/citest/images'
         keyring: /usr/share/keyrings/ubuntu-cloudimage-keyring.gpg
-        # The OS version formatted as Major.Minor is used to compare releases
-        version: null   # Each release needs to define this, for example 16.04
+        # The OS version formatted as Major.Minor is used to compare releases.
+        # Each release needs to define this, for example "16.04". Quoting is
+        # necessary to ensure the version is treated as a string.
+        version: null
 
     ec2:
         # Choose from: [ebs, instance-store]
@@ -131,12 +133,44 @@ features:
 
 releases:
     # UBUNTU =================================================================
+    groovy:
+        # EOL: Jul 2021
+        default:
+            enabled: true
+            release: groovy
+            version: "20.10"
+            os: ubuntu
+            feature_groups:
+                - base
+                - debian_base
+                - ubuntu_specific
+        lxd:
+            sstreams_server: https://cloud-images.ubuntu.com/daily
+            alias: groovy
+            setup_overrides: null
+            override_templates: false
+    focal:
+        # EOL: Apr 2025
+        default:
+            enabled: true
+            release: focal
+            version: "20.04"
+            os: ubuntu
+            feature_groups:
+                - base
+                - debian_base
+                - ubuntu_specific
+        lxd:
+            sstreams_server: https://cloud-images.ubuntu.com/daily
+            alias: focal
+            setup_overrides: null
+            override_templates: false
     eoan:
         # EOL: Jul 2020
         default:
             enabled: true
             release: eoan
-            version: 19.10
+            version: "19.10"
             os: ubuntu
             feature_groups:
                 - base
@@ -152,7 +186,7 @@ releases:
         default:
             enabled: true
             release: disco
-            version: 19.04
+            version: "19.04"
             os: ubuntu
             feature_groups:
                 - base
@@ -168,7 +202,7 @@ releases:
         default:
             enabled: true
             release: cosmic
-            version: 18.10
+            version: "18.10"
             os: ubuntu
             feature_groups:
                 - base
@@ -184,7 +218,7 @@ releases:
         default:
             enabled: true
             release: bionic
-            version: 18.04
+            version: "18.04"
             os: ubuntu
             feature_groups:
                 - base
@@ -200,7 +234,7 @@ releases:
         default:
             enabled: true
             release: artful
-            version: 17.10
+            version: "17.10"
             os: ubuntu
             feature_groups:
                 - base
@@ -216,7 +250,7 @@ releases:
         default:
             enabled: true
             release: xenial
-            version: 16.04
+            version: "16.04"
             os: ubuntu
             feature_groups:
                 - base
@@ -232,7 +266,7 @@ releases:
         default:
             enabled: true
             release: trusty
-            version: 14.04
+            version: "14.04"
             os: ubuntu
             feature_groups:
                 - base
diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py
index 6bb39f77..bb9785d3 100644
--- a/tests/cloud_tests/testcases/__init__.py
+++ b/tests/cloud_tests/testcases/__init__.py
@@ -4,7 +4,7 @@
 
 import importlib
 import inspect
-import unittest2
+import unittest
 
 from cloudinit.util import read_conf
 
@@ -21,8 +21,10 @@ def discover_test(test_name):
         config.name_sanitize(test_name))
     try:
         testmod = importlib.import_module(testmod_name)
-    except NameError:
-        raise ValueError('no test verifier found at: {}'.format(testmod_name))
+    except NameError as e:
+        raise ValueError(
+            'no test verifier found at: {}'.format(testmod_name)
+        ) from e
 
     found = [mod for name, mod in inspect.getmembers(testmod)
              if (inspect.isclass(mod)
@@ -48,7 +50,7 @@ def get_test_class(test_name, test_data, test_conf):
 
         def __str__(self):
             return "%s (%s)" % (self._testMethodName,
-                                unittest2.util.strclass(self._realclass))
+                                unittest.util.strclass(self._realclass))
 
         @classmethod
         def setUpClass(cls):
@@ -62,9 +64,9 @@ def get_suite(test_name, data, conf):
 
     @return_value: a test suite
     """
-    suite = unittest2.TestSuite()
+    suite = unittest.TestSuite()
     suite.addTest(
-        unittest2.defaultTestLoader.loadTestsFromTestCase(
+        unittest.defaultTestLoader.loadTestsFromTestCase(
             get_test_class(test_name, data, conf)))
     return suite
 
diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index fd12d87b..4448e0b5 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -5,15 +5,15 @@
 import crypt
 import json
 import re
-import unittest2
+import unittest
 
 
 from cloudinit import util as c_util
 
-SkipTest = unittest2.SkipTest
+SkipTest = unittest.SkipTest
 
 
-class CloudTestCase(unittest2.TestCase):
+class CloudTestCase(unittest.TestCase):
     """Base test class for verifiers."""
 
     # data gets populated in get_suite.setUpClass
@@ -34,7 +34,6 @@ class CloudTestCase(unittest2.TestCase):
     @classmethod
     def maybeSkipTest(cls):
         """Present to allow subclasses to override and raise a skipTest."""
-        pass
 
     def assertPackageInstalled(self, name, version=None):
         """Check dpkg-query --show output for matching package name.
@@ -141,8 +140,8 @@ class CloudTestCase(unittest2.TestCase):
     def test_no_warnings_in_log(self):
         """Unexpected warnings should not be found in the log."""
         warnings = [
-            l for l in self.get_data_file('cloud-init.log').splitlines()
-            if 'WARN' in l]
+            line for line in self.get_data_file('cloud-init.log').splitlines()
+            if 'WARN' in line]
         joined_warnings = '\n'.join(warnings)
         for expected_warning in self.expected_warnings:
             self.assertIn(
@@ -172,9 +171,7 @@ class CloudTestCase(unittest2.TestCase):
                 'Skipping instance-data.json test.'
                 ' OS: %s not bionic or newer' % self.os_name)
         instance_data = json.loads(out)
-        self.assertItemsEqual(
-            [],
-            instance_data['base64_encoded_keys'])
+        self.assertCountEqual(['merged_cfg'], instance_data['sensitive_keys'])
         ds = instance_data.get('ds', {})
         v1_data = instance_data.get('v1', {})
         metadata = ds.get('meta-data', {})
@@ -201,6 +198,23 @@ class CloudTestCase(unittest2.TestCase):
         self.assertIn('i-', v1_data['instance_id'])
         self.assertIn('ip-', v1_data['local_hostname'])
         self.assertIsNotNone(v1_data['region'], 'expected ec2 region')
+        self.assertIsNotNone(
+            re.match(r'\d\.\d+\.\d+-\d+-aws', v1_data['kernel_release']))
+        self.assertEqual(
+            'redacted for non-root user', instance_data['merged_cfg'])
+        self.assertEqual(self.os_cfg['os'], v1_data['variant'])
+        self.assertEqual(self.os_cfg['os'], v1_data['distro'])
+        self.assertEqual(
+            self.os_cfg['os'], instance_data["sys_info"]['dist'][0],
+            "Unexpected sys_info dist value")
+        self.assertEqual(self.os_name, v1_data['distro_release'])
+        self.assertEqual(
+            str(self.os_cfg['version']), v1_data['distro_version'])
+        self.assertEqual('x86_64', v1_data['machine'])
+        self.assertIsNotNone(
+            re.match(r'3.\d\.\d', v1_data['python_version']),
+            "unexpected python version: {ver}".format(
+                ver=v1_data["python_version"]))
 
     def test_instance_data_json_lxd(self):
         """Validate instance-data.json content by lxd platform.
@@ -222,7 +236,7 @@ class CloudTestCase(unittest2.TestCase):
                 ' OS: %s not bionic or newer' % self.os_name)
         instance_data = json.loads(out)
         v1_data = instance_data.get('v1', {})
-        self.assertItemsEqual([], sorted(instance_data['base64_encoded_keys']))
+        self.assertCountEqual([], sorted(instance_data['base64_encoded_keys']))
         self.assertEqual('unknown', v1_data['cloud_name'])
         self.assertEqual('lxd', v1_data['platform'])
         self.assertEqual(
@@ -237,6 +251,23 @@ class CloudTestCase(unittest2.TestCase):
         self.assertIsNone(
             v1_data['region'],
             'found unexpected lxd region %s' % v1_data['region'])
+        self.assertIsNotNone(
+            re.match(r'\d\.\d+\.\d+-\d+', v1_data['kernel_release']))
+        self.assertEqual(
+            'redacted for non-root user', instance_data['merged_cfg'])
+        self.assertEqual(self.os_cfg['os'], v1_data['variant'])
+        self.assertEqual(self.os_cfg['os'], v1_data['distro'])
+        self.assertEqual(
+            self.os_cfg['os'], instance_data["sys_info"]['dist'][0],
+            "Unexpected sys_info dist value")
+        self.assertEqual(self.os_name, v1_data['distro_release'])
+        self.assertEqual(
+            str(self.os_cfg['version']), v1_data['distro_version'])
+        self.assertEqual('x86_64', v1_data['machine'])
+        self.assertIsNotNone(
+            re.match(r'3.\d\.\d', v1_data['python_version']),
+            "unexpected python version: {ver}".format(
+                ver=v1_data["python_version"]))
 
     def test_instance_data_json_kvm(self):
         """Validate instance-data.json content by nocloud-kvm platform.
@@ -259,7 +290,7 @@ class CloudTestCase(unittest2.TestCase):
                 ' OS: %s not bionic or newer' % self.os_name)
         instance_data = json.loads(out)
         v1_data = instance_data.get('v1', {})
-        self.assertItemsEqual([], instance_data['base64_encoded_keys'])
+        self.assertCountEqual([], instance_data['base64_encoded_keys'])
         self.assertEqual('unknown', v1_data['cloud_name'])
         self.assertEqual('nocloud', v1_data['platform'])
         subplatform = v1_data['subplatform']
@@ -278,6 +309,23 @@ class CloudTestCase(unittest2.TestCase):
         self.assertIsNone(
             v1_data['region'],
             'found unexpected lxd region %s' % v1_data['region'])
+        self.assertIsNotNone(
+            re.match(r'\d\.\d+\.\d+-\d+', v1_data['kernel_release']))
+        self.assertEqual(
+            'redacted for non-root user', instance_data['merged_cfg'])
+        self.assertEqual(self.os_cfg['os'], v1_data['variant'])
+        self.assertEqual(self.os_cfg['os'], v1_data['distro'])
+        self.assertEqual(
+            self.os_cfg['os'], instance_data["sys_info"]['dist'][0],
+            "Unexpected sys_info dist value")
+        self.assertEqual(self.os_name, v1_data['distro_release'])
+        self.assertEqual(
+            str(self.os_cfg['version']), v1_data['distro_version'])
+        self.assertEqual('x86_64', v1_data['machine'])
+        self.assertIsNotNone(
+            re.match(r'3.\d\.\d', v1_data['python_version']),
+            "unexpected python version: {ver}".format(
+                ver=v1_data["python_version"]))
 
 
 class PasswordListTest(CloudTestCase):
diff --git a/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml b/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml
index 0bec305e..68ca95b5 100644
--- a/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml
+++ b/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml
@@ -8,43 +8,44 @@ cloud_config: |
   #cloud-config
   # Key from https://packages.chef.io/chef.asc
   apt:
-    source1:
-      source: "deb http://packages.chef.io/repos/apt/stable $RELEASE main"
-      key: |
-        -----BEGIN PGP PUBLIC KEY BLOCK-----
-        Version: GnuPG v1.4.12 (Darwin)
-        Comment: GPGTools - http://gpgtools.org
+    sources:
+      source1:
+        source: "deb http://packages.chef.io/repos/apt/stable $RELEASE main"
+        key: |
+          -----BEGIN PGP PUBLIC KEY BLOCK-----
+          Version: GnuPG v1.4.12 (Darwin)
+          Comment: GPGTools - http://gpgtools.org
 
-        mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu
-        twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99
-        dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC
-        JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W
-        ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I
-        XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe
-        DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm
-        sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO
-        Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ
-        YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG
-        CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K
-        +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu0IENIRUYgUGFja2FnZXMg
-        PHBhY2thZ2VzQGNoZWYuaW8+iGIEExECACIFAlQwYFECGwMGCwkIBwMCBhUIAgkK
-        CwQWAgMBAh4BAheAAAoJEClAq6mD74JqX94An26z99XOHWpLN8ahzm7cp13t4Xid
-        AJ9wVcgoUBzvgg91lKfv/34cmemZn7kCDQRKaQu0EAgAg7ZLCVGVTmLqBM6njZEd
-        Zbv+mZbvwLBSomdiqddE6u3eH0X3GuwaQfQWHUVG2yedyDMiG+EMtCdEeeRebTCz
-        SNXQ8Xvi22hRPoEsBSwWLZI8/XNg0n0f1+GEr+mOKO0BxDB2DG7DA0nnEISxwFkK
-        OFJFebR3fRsrWjj0KjDxkhse2ddU/jVz1BY7Nf8toZmwpBmdozETMOTx3LJy1HZ/
-        Te9FJXJMUaB2lRyluv15MVWCKQJro4MQG/7QGcIfrIZNfAGJ32DDSjV7/YO+IpRY
-        IL4CUBQ65suY4gYUG4jhRH6u7H1p99sdwsg5OIpBe/v2Vbc/tbwAB+eJJAp89Zeu
-        twADBQf/ZcGoPhTGFuzbkcNRSIz+boaeWPoSxK2DyfScyCAuG41CY9+g0HIw9Sq8
-        DuxQvJ+vrEJjNvNE3EAEdKl/zkXMZDb1EXjGwDi845TxEMhhD1dDw2qpHqnJ2mtE
-        WpZ7juGwA3sGhi6FapO04tIGacCfNNHmlRGipyq5ZiKIRq9mLEndlECr8cwaKgkS
-        0wWu+xmMZe7N5/t/TK19HXNh4tVacv0F3fYK54GUjt2FjCQV75USnmNY4KPTYLXA
-        dzC364hEMlXpN21siIFgB04w+TXn5UF3B4FfAy5hevvr4DtV4MvMiGLu0oWjpaLC
-        MpmrR3Ny2wkmO0h+vgri9uIP06ODWIhJBBgRAgAJBQJKaQu0AhsMAAoJEClAq6mD
-        74Jq4hIAoJ5KrYS8kCwj26SAGzglwggpvt3CAJ0bekyky56vNqoegB+y4PQVDv4K
-        zA==
-        =IxPr
-        -----END PGP PUBLIC KEY BLOCK-----
+          mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu
+          twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99
+          dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC
+          JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W
+          ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I
+          XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe
+          DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm
+          sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO
+          Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ
+          YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG
+          CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K
+          +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu0IENIRUYgUGFja2FnZXMg
+          PHBhY2thZ2VzQGNoZWYuaW8+iGIEExECACIFAlQwYFECGwMGCwkIBwMCBhUIAgkK
+          CwQWAgMBAh4BAheAAAoJEClAq6mD74JqX94An26z99XOHWpLN8ahzm7cp13t4Xid
+          AJ9wVcgoUBzvgg91lKfv/34cmemZn7kCDQRKaQu0EAgAg7ZLCVGVTmLqBM6njZEd
+          Zbv+mZbvwLBSomdiqddE6u3eH0X3GuwaQfQWHUVG2yedyDMiG+EMtCdEeeRebTCz
+          SNXQ8Xvi22hRPoEsBSwWLZI8/XNg0n0f1+GEr+mOKO0BxDB2DG7DA0nnEISxwFkK
+          OFJFebR3fRsrWjj0KjDxkhse2ddU/jVz1BY7Nf8toZmwpBmdozETMOTx3LJy1HZ/
+          Te9FJXJMUaB2lRyluv15MVWCKQJro4MQG/7QGcIfrIZNfAGJ32DDSjV7/YO+IpRY
+          IL4CUBQ65suY4gYUG4jhRH6u7H1p99sdwsg5OIpBe/v2Vbc/tbwAB+eJJAp89Zeu
+          twADBQf/ZcGoPhTGFuzbkcNRSIz+boaeWPoSxK2DyfScyCAuG41CY9+g0HIw9Sq8
+          DuxQvJ+vrEJjNvNE3EAEdKl/zkXMZDb1EXjGwDi845TxEMhhD1dDw2qpHqnJ2mtE
+          WpZ7juGwA3sGhi6FapO04tIGacCfNNHmlRGipyq5ZiKIRq9mLEndlECr8cwaKgkS
+          0wWu+xmMZe7N5/t/TK19HXNh4tVacv0F3fYK54GUjt2FjCQV75USnmNY4KPTYLXA
+          dzC364hEMlXpN21siIFgB04w+TXn5UF3B4FfAy5hevvr4DtV4MvMiGLu0oWjpaLC
+          MpmrR3Ny2wkmO0h+vgri9uIP06ODWIhJBBgRAgAJBQJKaQu0AhsMAAoJEClAq6mD
+          74Jq4hIAoJ5KrYS8kCwj26SAGzglwggpvt3CAJ0bekyky56vNqoegB+y4PQVDv4K
+          zA==
+          =IxPr
+          -----END PGP PUBLIC KEY BLOCK-----
 
   chef:
 
diff --git a/tests/cloud_tests/testcases/modules/ntp_chrony.py b/tests/cloud_tests/testcases/modules/ntp_chrony.py
index 0f4c3d08..7d341773 100644
--- a/tests/cloud_tests/testcases/modules/ntp_chrony.py
+++ b/tests/cloud_tests/testcases/modules/ntp_chrony.py
@@ -1,7 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 """cloud-init Integration Test Verify Script."""
-import unittest2
+import unittest
 
 from tests.cloud_tests.testcases import base
 
@@ -13,7 +13,7 @@ class TestNtpChrony(base.CloudTestCase):
         """Skip this suite of tests on lxd and artful or older."""
         if self.platform == 'lxd':
             if self.is_distro('ubuntu') and self.os_version_cmp('artful') <= 0:
-                raise unittest2.SkipTest(
+                raise unittest.SkipTest(
                     'No support for chrony on containers <= artful.'
                     ' LP: #1589780')
         return super(TestNtpChrony, self).setUp()
diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py
index 06f7d865..7dcccbdd 100644
--- a/tests/cloud_tests/util.py
+++ b/tests/cloud_tests/util.py
@@ -5,6 +5,7 @@
 import base64
 import copy
 import glob
+import multiprocessing
 import os
 import random
 import shlex
@@ -12,8 +13,11 @@ import shutil
 import string
 import subprocess
 import tempfile
+import time
 import yaml
+from contextlib import contextmanager
 
+from cloudinit import subp
 from cloudinit import util as c_util
 from tests.cloud_tests import LOG
 
@@ -118,6 +122,36 @@ def current_verbosity():
     return max(min(3 - int(LOG.level / 10), 2), 0)
 
 
+@contextmanager
+def emit_dots_on_travis():
+    """
+    A context manager that emits a dot every 10 seconds if running on Travis.
+
+    Travis will kill jobs that don't emit output for a certain amount of time.
+    This context manager spins up a background process which will emit a dot to
+    stdout every 10 seconds to avoid being killed.
+
+    It should be wrapped selectively around operations that are known to take a
+    long time.
+    """
+    if os.environ.get('TRAVIS') != "true":
+        # If we aren't on Travis, don't do anything.
+        yield
+        return
+
+    def emit_dots():
+        while True:
+            print(".")
+            time.sleep(10)
+
+    dot_process = multiprocessing.Process(target=emit_dots)
+    dot_process.start()
+    try:
+        yield
+    finally:
+        dot_process.terminate()
+
+
 def is_writable_dir(path):
     """Make sure dir is writable.
 
@@ -199,8 +233,8 @@ def flat_tar(output, basedir, owner='root', group='root'):
     @param group: group archive files belong to
     @return_value: none
     """
-    c_util.subp(['tar', 'cf', output, '--owner', owner, '--group', group,
-                 '-C', basedir] + rel_files(basedir), capture=True)
+    subp.subp(['tar', 'cf', output, '--owner', owner, '--group', group,
+               '-C', basedir] + rel_files(basedir), capture=True)
 
 
 def parse_conf_list(entries, valid=None, boolean=False):
@@ -432,7 +466,7 @@ class TargetBase(object):
         return path
 
 
-class InTargetExecuteError(c_util.ProcessExecutionError):
+class InTargetExecuteError(subp.ProcessExecutionError):
     """Error type for in target commands that fail."""
 
     default_desc = 'Unexpected error while running command.'
diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py
index 7018f4d5..0295af40 100644
--- a/tests/cloud_tests/verify.py
+++ b/tests/cloud_tests/verify.py
@@ -3,7 +3,7 @@
 """Verify test results."""
 
 import os
-import unittest2
+import unittest
 
 from tests.cloud_tests import (config, LOG, util, testcases)
 
@@ -18,7 +18,7 @@ def verify_data(data_dir, platform, os_name, tests):
     @return_value: {<test_name>: {passed: True/False, failures: []}}
     """
     base_dir = os.sep.join((data_dir, platform, os_name))
-    runner = unittest2.TextTestRunner(verbosity=util.current_verbosity())
+    runner = unittest.TextTestRunner(verbosity=util.current_verbosity())
     res = {}
     for test_name in tests:
         LOG.debug('verifying test data for %s', test_name)