diff options
Diffstat (limited to 'tests/cloud_tests')
30 files changed, 280 insertions, 78 deletions
diff --git a/tests/cloud_tests/args.py b/tests/cloud_tests/args.py index c6c1877b..ab345491 100644 --- a/tests/cloud_tests/args.py +++ b/tests/cloud_tests/args.py @@ -62,6 +62,9 @@ ARG_SETS = { (('-d', '--data-dir'), {'help': 'directory to store test data in', 'action': 'store', 'metavar': 'DIR', 'required': False}), + (('--preserve-instance',), + {'help': 'do not destroy the instance under test', + 'action': 'store_true', 'default': False, 'required': False}), (('--preserve-data',), {'help': 'do not remove collected data after successful run', 'action': 'store_true', 'default': False, 'required': False}),), diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py index b9cfcfa6..f04d0cd4 100644 --- a/tests/cloud_tests/bddeb.py +++ b/tests/cloud_tests/bddeb.py @@ -113,7 +113,7 @@ def bddeb(args): @return_value: fail count """ LOG.info('preparing to build cloud-init deb') - (res, failed) = run_stage('build deb', [partial(setup_build, args)]) + _res, failed = run_stage('build deb', [partial(setup_build, args)]) return failed # vi: ts=4 expandtab diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index d4f9135b..75b50616 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -25,7 +25,8 @@ def collect_script(instance, base_dir, script, script_name): script.encode(), rcs=False, description='collect: {}'.format(script_name)) if err: - LOG.debug("collect script %s had stderr: %s", script_name, err) + LOG.debug("collect script %s exited '%s' and had stderr: %s", + script_name, err, exit) if not isinstance(out, bytes): raise util.PlatformError( "Collection of '%s' returned type %s, expected bytes: %s" % @@ -41,7 +42,7 @@ def collect_console(instance, base_dir): @param base_dir: directory to write console log to """ logfile = os.path.join(base_dir, 'console.log') - LOG.debug('getting console log for %s to %s', instance, logfile) + LOG.debug('getting console log for %s to %s', instance.name, logfile) try: data = instance.console_log() except NotImplementedError as e: @@ -92,7 +93,8 @@ def collect_test_data(args, snapshot, os_name, test_name): # create test instance component = PlatformComponent( partial(platforms.get_instance, snapshot, user_data, - block=True, start=False, use_desc=test_name)) + block=True, start=False, use_desc=test_name), + preserve_instance=args.preserve_instance) LOG.info('collecting test data for test: %s', test_name) with component as instance: diff --git a/tests/cloud_tests/platforms/instances.py b/tests/cloud_tests/platforms/instances.py index 3bad021f..95bc3b16 100644 --- a/tests/cloud_tests/platforms/instances.py +++ b/tests/cloud_tests/platforms/instances.py @@ -87,7 +87,12 @@ class Instance(TargetBase): self._ssh_client = None def _ssh_connect(self): - """Connect via SSH.""" + """Connect via SSH. + + Attempt to SSH to the client on the specific IP and port. If it + fails in some manner, then retry 2 more times for a total of 3 + attempts; sleeping a few seconds between attempts. + """ if self._ssh_client: return self._ssh_client @@ -98,21 +103,22 @@ class Instance(TargetBase): client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) private_key = paramiko.RSAKey.from_private_key_file(self.ssh_key_file) - retries = 30 + retries = 3 while retries: try: client.connect(username=self.ssh_username, hostname=self.ssh_ip, port=self.ssh_port, - pkey=private_key, banner_timeout=30) + pkey=private_key) self._ssh_client = client return client except (ConnectionRefusedError, AuthenticationException, BadHostKeyException, ConnectionResetError, SSHException, - OSError) as e: + OSError): retries -= 1 - time.sleep(10) + LOG.debug('Retrying ssh connection on connect failure') + time.sleep(3) - ssh_cmd = 'Failed ssh connection to %s@%s:%s after 300 seconds' % ( + ssh_cmd = 'Failed ssh connection to %s@%s:%s after 3 retries' % ( self.ssh_username, self.ssh_ip, self.ssh_port ) raise util.InTargetExecuteError(b'', b'', 1, ssh_cmd, 'ssh') @@ -128,18 +134,31 @@ class Instance(TargetBase): return ' '.join(l for l in test.strip().splitlines() if not l.lstrip().startswith('#')) - time = self.config['boot_timeout'] + boot_timeout = self.config['boot_timeout'] tests = [self.config['system_ready_script']] if wait_for_cloud_init: tests.append(self.config['cloud_init_ready_script']) formatted_tests = ' && '.join(clean_test(t) for t in tests) cmd = ('i=0; while [ $i -lt {time} ] && i=$(($i+1)); do {test} && ' - 'exit 0; sleep 1; done; exit 1').format(time=time, + 'exit 0; sleep 1; done; exit 1').format(time=boot_timeout, test=formatted_tests) - if self.execute(cmd, rcs=(0, 1))[-1] != 0: - raise OSError('timeout: after {}s system not started'.format(time)) - + end_time = time.time() + boot_timeout + while True: + try: + return_code = self.execute( + cmd, rcs=(0, 1), description='wait for instance start' + )[-1] + if return_code == 0: + break + except util.InTargetExecuteError: + LOG.warning("failed to connect via SSH") + + if time.time() < end_time: + time.sleep(3) + else: + raise util.PlatformError('ssh', 'after %ss instance is not ' + 'reachable' % boot_timeout) # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py index 0d957bca..d396519f 100644 --- a/tests/cloud_tests/platforms/lxd/instance.py +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -152,9 +152,8 @@ class LXDInstance(Instance): return fp.read() try: - stdout, stderr = subp( - ['lxc', 'console', '--show-log', self.name], decode=False) - return stdout + return subp(['lxc', 'console', '--show-log', self.name], + decode=False)[0] except ProcessExecutionError as e: raise PlatformError( "console log", @@ -209,16 +208,15 @@ def _has_proper_console_support(): if 'console' not in info.get('api_extensions', []): reason = "LXD server does not support console api extension" else: - dver = info.get('environment', {}).get('driver_version', "") + dver = str(info.get('environment', {}).get('driver_version', "")) if dver.startswith("2.") or dver.startswith("1."): reason = "LXD Driver version not 3.x+ (%s)" % dver else: try: - stdout, stderr = subp(['lxc', 'console', '--help'], - decode=False) + stdout = subp(['lxc', 'console', '--help'], decode=False)[0] if not (b'console' in stdout and b'log' in stdout): reason = "no '--log' in lxc console --help" - except ProcessExecutionError as e: + except ProcessExecutionError: reason = "no 'console' command in lxc client" if reason: diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index c7dcbe83..defae02b 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -129,6 +129,22 @@ features: releases: # UBUNTU ================================================================= + cosmic: + # EOL: Jul 2019 + default: + enabled: true + release: cosmic + version: 18.10 + os: ubuntu + feature_groups: + - base + - debian_base + - ubuntu_specific + lxd: + sstreams_server: https://cloud-images.ubuntu.com/daily + alias: cosmic + setup_overrides: null + override_templates: false bionic: # EOL: Apr 2023 default: diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py index 6d242115..4e195709 100644 --- a/tests/cloud_tests/setup_image.py +++ b/tests/cloud_tests/setup_image.py @@ -25,10 +25,9 @@ def installed_package_version(image, package, ensure_installed=True): else: raise NotImplementedError - msg = 'query version for package: {}'.format(package) - (out, err, exit) = image.execute( - cmd, description=msg, rcs=(0,) if ensure_installed else range(0, 256)) - return out.strip() + return image.execute( + cmd, description='query version for package: {}'.format(package), + rcs=(0,) if ensure_installed else range(0, 256))[0].strip() def install_deb(args, image): @@ -54,7 +53,7 @@ def install_deb(args, image): remote_path], description=msg) # check installed deb version matches package fmt = ['-W', "--showformat=${Version}"] - (out, err, exit) = image.execute(['dpkg-deb'] + fmt + [remote_path]) + out = image.execute(['dpkg-deb'] + fmt + [remote_path])[0] expected_version = out.strip() found_version = installed_package_version(image, 'cloud-init') if expected_version != found_version: @@ -85,7 +84,7 @@ def install_rpm(args, image): image.execute(['rpm', '-U', remote_path], description=msg) fmt = ['--queryformat', '"%{VERSION}"'] - (out, err, exit) = image.execute(['rpm', '-q'] + fmt + [remote_path]) + (out, _err, _exit) = image.execute(['rpm', '-q'] + fmt + [remote_path]) expected_version = out.strip() found_version = installed_package_version(image, 'cloud-init') if expected_version != found_version: diff --git a/tests/cloud_tests/stage.py b/tests/cloud_tests/stage.py index 74a7d46d..d64a1dcc 100644 --- a/tests/cloud_tests/stage.py +++ b/tests/cloud_tests/stage.py @@ -12,9 +12,15 @@ from tests.cloud_tests import LOG class PlatformComponent(object): """Context manager to safely handle platform components.""" - def __init__(self, get_func): - """Store get_<platform component> function as partial with no args.""" + def __init__(self, get_func, preserve_instance=False): + """Store get_<platform component> function as partial with no args. + + @param get_func: Callable returning an instance from the platform. + @param preserve_instance: Boolean, when True, do not destroy instance + after test. Used for test development. + """ self.get_func = get_func + self.preserve_instance = preserve_instance def __enter__(self): """Create instance of platform component.""" @@ -24,7 +30,10 @@ class PlatformComponent(object): def __exit__(self, etype, value, trace): """Destroy instance.""" if self.instance is not None: - self.instance.destroy() + if self.preserve_instance: + LOG.info('Preserving test instance %s', self.instance.name) + else: + self.instance.destroy() def run_single(name, call): diff --git a/tests/cloud_tests/testcases.yaml b/tests/cloud_tests/testcases.yaml index a3e29900..a16d1ddf 100644 --- a/tests/cloud_tests/testcases.yaml +++ b/tests/cloud_tests/testcases.yaml @@ -24,9 +24,9 @@ base_test_data: status.json: | #!/bin/sh cat /run/cloud-init/status.json - cloud-init-version: | + package-versions: | #!/bin/sh - dpkg-query -W -f='${Version}' cloud-init + dpkg-query --show system.journal.gz: | #!/bin/sh [ -d /run/systemd ] || { echo "not systemd."; exit 0; } diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 324c7c91..696db8dd 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -31,6 +31,27 @@ class CloudTestCase(unittest.TestCase): def is_distro(self, distro_name): return self.os_cfg['os'] == distro_name + def assertPackageInstalled(self, name, version=None): + """Check dpkg-query --show output for matching package name. + + @param name: package base name + @param version: string representing a package version or part of a + version. + """ + pkg_out = self.get_data_file('package-versions') + pkg_match = re.search( + '^%s\t(?P<version>.*)$' % name, pkg_out, re.MULTILINE) + if pkg_match: + installed_version = pkg_match.group('version') + if not version: + return # Success + if installed_version.startswith(version): + return # Success + raise AssertionError( + 'Expected package version %s-%s not found. Found %s' % + name, version, installed_version) + raise AssertionError('Package not installed: %s' % name) + def os_version_cmp(self, cmp_version): """Compare the version of the test to comparison_version. @@ -149,21 +170,22 @@ class CloudTestCase(unittest.TestCase): self.assertEqual( ['ds/user-data'], instance_data['base64-encoded-keys']) ds = instance_data.get('ds', {}) - macs = ds.get('network', {}).get('interfaces', {}).get('macs', {}) + v1_data = instance_data.get('v1', {}) + metadata = ds.get('meta-data', {}) + macs = metadata.get( + 'network', {}).get('interfaces', {}).get('macs', {}) if not macs: raise AssertionError('No network data from EC2 meta-data') # Check meta-data items we depend on expected_net_keys = [ 'public-ipv4s', 'ipv4-associations', 'local-hostname', 'public-hostname'] - for mac, mac_data in macs.items(): + for mac_data in macs.values(): for key in expected_net_keys: self.assertIn(key, mac_data) self.assertIsNotNone( - ds.get('placement', {}).get('availability-zone'), + metadata.get('placement', {}).get('availability-zone'), 'Could not determine EC2 Availability zone placement') - ds = instance_data.get('ds', {}) - v1_data = instance_data.get('v1', {}) self.assertIsNotNone( v1_data['availability-zone'], 'expected ec2 availability-zone') self.assertEqual('aws', v1_data['cloud-name']) @@ -234,7 +256,7 @@ class CloudTestCase(unittest.TestCase): 'found unexpected kvm availability-zone %s' % v1_data['availability-zone']) self.assertIsNotNone( - re.match('[\da-f]{8}(-[\da-f]{4}){3}-[\da-f]{12}', + re.match(r'[\da-f]{8}(-[\da-f]{4}){3}-[\da-f]{12}', v1_data['instance-id']), 'kvm instance-id is not a UUID: %s' % v1_data['instance-id']) self.assertIn('ubuntu', v1_data['local-hostname']) diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.py b/tests/cloud_tests/testcases/examples/including_user_groups.py index 93b7a82d..4067348d 100644 --- a/tests/cloud_tests/testcases/examples/including_user_groups.py +++ b/tests/cloud_tests/testcases/examples/including_user_groups.py @@ -42,7 +42,7 @@ class TestUserGroups(base.CloudTestCase): def test_user_root_in_secret(self): """Test root user is in 'secret' group.""" - user, _, groups = self.get_data_file('root_groups').partition(":") + _user, _, groups = self.get_data_file('root_groups').partition(":") self.assertIn("secret", groups.split(), msg="User root is not in group 'secret'") diff --git a/tests/cloud_tests/testcases/modules/byobu.py b/tests/cloud_tests/testcases/modules/byobu.py index 005ca014..74d0529a 100644 --- a/tests/cloud_tests/testcases/modules/byobu.py +++ b/tests/cloud_tests/testcases/modules/byobu.py @@ -9,8 +9,7 @@ class TestByobu(base.CloudTestCase): def test_byobu_installed(self): """Test byobu installed.""" - out = self.get_data_file('byobu_installed') - self.assertIn('/usr/bin/byobu', out) + self.assertPackageInstalled('byobu') def test_byobu_profile_enabled(self): """Test byobu profile.d file exists.""" diff --git a/tests/cloud_tests/testcases/modules/byobu.yaml b/tests/cloud_tests/testcases/modules/byobu.yaml index a9aa1f3f..d002a611 100644 --- a/tests/cloud_tests/testcases/modules/byobu.yaml +++ b/tests/cloud_tests/testcases/modules/byobu.yaml @@ -7,9 +7,6 @@ cloud_config: | #cloud-config byobu_by_default: enable collect_scripts: - byobu_installed: | - #!/bin/bash - which byobu byobu_profile_enabled: | #!/bin/bash ls /etc/profile.d/Z97-byobu.sh diff --git a/tests/cloud_tests/testcases/modules/ca_certs.py b/tests/cloud_tests/testcases/modules/ca_certs.py index e75f0413..6b56f639 100644 --- a/tests/cloud_tests/testcases/modules/ca_certs.py +++ b/tests/cloud_tests/testcases/modules/ca_certs.py @@ -7,10 +7,23 @@ from tests.cloud_tests.testcases import base class TestCaCerts(base.CloudTestCase): """Test ca certs module.""" - def test_cert_count(self): - """Test the count is proper.""" - out = self.get_data_file('cert_count') - self.assertEqual(5, int(out)) + def test_certs_updated(self): + """Test certs have been updated in /etc/ssl/certs.""" + out = self.get_data_file('cert_links') + # Bionic update-ca-certificates creates less links debian #895075 + unlinked_files = [] + links = {} + for cert_line in out.splitlines(): + if '->' in cert_line: + fname, _sep, link = cert_line.split() + links[fname] = link + else: + unlinked_files.append(cert_line) + self.assertEqual(['ca-certificates.crt'], unlinked_files) + self.assertEqual('cloud-init-ca-certs.pem', links['a535c1f3.0']) + self.assertEqual( + '/usr/share/ca-certificates/cloud-init-ca-certs.crt', + links['cloud-init-ca-certs.pem']) def test_cert_installed(self): """Test line from our cert exists.""" diff --git a/tests/cloud_tests/testcases/modules/ca_certs.yaml b/tests/cloud_tests/testcases/modules/ca_certs.yaml index d939f435..2cd91551 100644 --- a/tests/cloud_tests/testcases/modules/ca_certs.yaml +++ b/tests/cloud_tests/testcases/modules/ca_certs.yaml @@ -43,9 +43,13 @@ cloud_config: | DiH5uEqBXExjrj0FslxcVKdVj5glVcSmkLwZKbEU1OKwleT/iXFhvooWhQ== -----END CERTIFICATE----- collect_scripts: - cert_count: | + cert_links: | #!/bin/bash - ls -l /etc/ssl/certs | wc -l + # links printed <filename> -> <link target> + # non-links printed <filename> + for file in `ls /etc/ssl/certs`; do + [ -h /etc/ssl/certs/$file ] && echo -n $file ' -> ' && readlink /etc/ssl/certs/$file || echo $file; + done cert: | #!/bin/bash md5sum /etc/ssl/certs/ca-certificates.crt diff --git a/tests/cloud_tests/testcases/modules/ntp.py b/tests/cloud_tests/testcases/modules/ntp.py index b50e52fe..c63cc15e 100644 --- a/tests/cloud_tests/testcases/modules/ntp.py +++ b/tests/cloud_tests/testcases/modules/ntp.py @@ -9,15 +9,14 @@ class TestNtp(base.CloudTestCase): def test_ntp_installed(self): """Test ntp installed""" - out = self.get_data_file('ntp_installed') - self.assertEqual(0, int(out)) + self.assertPackageInstalled('ntp') def test_ntp_dist_entries(self): """Test dist config file is empty""" out = self.get_data_file('ntp_conf_dist_empty') self.assertEqual(0, int(out)) - def test_ntp_entires(self): + def test_ntp_entries(self): """Test config entries""" out = self.get_data_file('ntp_conf_pool_list') self.assertIn('pool.ntp.org iburst', out) diff --git a/tests/cloud_tests/testcases/modules/ntp.yaml b/tests/cloud_tests/testcases/modules/ntp.yaml index 2530d72e..7ea0707d 100644 --- a/tests/cloud_tests/testcases/modules/ntp.yaml +++ b/tests/cloud_tests/testcases/modules/ntp.yaml @@ -4,6 +4,7 @@ cloud_config: | #cloud-config ntp: + ntp_client: ntp pools: [] servers: [] collect_scripts: diff --git a/tests/cloud_tests/testcases/modules/ntp_chrony.py b/tests/cloud_tests/testcases/modules/ntp_chrony.py new file mode 100644 index 00000000..7d341773 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp_chrony.py @@ -0,0 +1,26 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""cloud-init Integration Test Verify Script.""" +import unittest + +from tests.cloud_tests.testcases import base + + +class TestNtpChrony(base.CloudTestCase): + """Test ntp module with chrony client""" + + def setUp(self): + """Skip this suite of tests on lxd and artful or older.""" + if self.platform == 'lxd': + if self.is_distro('ubuntu') and self.os_version_cmp('artful') <= 0: + raise unittest.SkipTest( + 'No support for chrony on containers <= artful.' + ' LP: #1589780') + return super(TestNtpChrony, self).setUp() + + def test_chrony_entries(self): + """Test chrony config entries""" + out = self.get_data_file('chrony_conf') + self.assertIn('.pool.ntp.org', out) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ntp_chrony.yaml b/tests/cloud_tests/testcases/modules/ntp_chrony.yaml new file mode 100644 index 00000000..120735e2 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp_chrony.yaml @@ -0,0 +1,17 @@ +# +# ntp enabled, chrony selected, check conf file +# as chrony won't start in a container +# +cloud_config: | + #cloud-config + ntp: + enabled: true + ntp_client: chrony +collect_scripts: + chrony_conf: | + #!/bin/sh + set -- /etc/chrony.conf /etc/chrony/chrony.conf + for p in "$@"; do + [ -e "$p" ] && { cat "$p"; exit; } + done +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ntp_pools.yaml b/tests/cloud_tests/testcases/modules/ntp_pools.yaml index d490b228..60fa0fd1 100644 --- a/tests/cloud_tests/testcases/modules/ntp_pools.yaml +++ b/tests/cloud_tests/testcases/modules/ntp_pools.yaml @@ -9,6 +9,7 @@ required_features: cloud_config: | #cloud-config ntp: + ntp_client: ntp pools: - 0.cloud-init.mypool - 1.cloud-init.mypool diff --git a/tests/cloud_tests/testcases/modules/ntp_servers.yaml b/tests/cloud_tests/testcases/modules/ntp_servers.yaml index 6b13b70e..ee636679 100644 --- a/tests/cloud_tests/testcases/modules/ntp_servers.yaml +++ b/tests/cloud_tests/testcases/modules/ntp_servers.yaml @@ -6,6 +6,7 @@ required_features: cloud_config: | #cloud-config ntp: + ntp_client: ntp servers: - 172.16.15.14 - 172.16.17.18 diff --git a/tests/cloud_tests/testcases/modules/ntp_timesyncd.py b/tests/cloud_tests/testcases/modules/ntp_timesyncd.py new file mode 100644 index 00000000..eca750bc --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp_timesyncd.py @@ -0,0 +1,15 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""cloud-init Integration Test Verify Script.""" +from tests.cloud_tests.testcases import base + + +class TestNtpTimesyncd(base.CloudTestCase): + """Test ntp module with systemd-timesyncd client""" + + def test_timesyncd_entries(self): + """Test timesyncd config entries""" + out = self.get_data_file('timesyncd_conf') + self.assertIn('.pool.ntp.org', out) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ntp_timesyncd.yaml b/tests/cloud_tests/testcases/modules/ntp_timesyncd.yaml new file mode 100644 index 00000000..ee47a741 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp_timesyncd.yaml @@ -0,0 +1,15 @@ +# +# ntp enabled, systemd-timesyncd selected, check conf file +# as systemd-timesyncd won't start in a container +# +cloud_config: | + #cloud-config + ntp: + enabled: true + ntp_client: systemd-timesyncd +collect_scripts: + timesyncd_conf: | + #!/bin/sh + cat /etc/systemd/timesyncd.conf.d/cloud-init.conf + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py index a92dec22..fecad768 100644 --- a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py +++ b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py @@ -7,15 +7,13 @@ from tests.cloud_tests.testcases import base class TestPackageInstallUpdateUpgrade(base.CloudTestCase): """Test package install update upgrade module.""" - def test_installed_htop(self): - """Test htop got installed.""" - out = self.get_data_file('dpkg_htop') - self.assertEqual(1, int(out)) + def test_installed_sl(self): + """Test sl got installed.""" + self.assertPackageInstalled('sl') def test_installed_tree(self): """Test tree got installed.""" - out = self.get_data_file('dpkg_tree') - self.assertEqual(1, int(out)) + self.assertPackageInstalled('tree') def test_apt_history(self): """Test apt history for update command.""" @@ -23,13 +21,13 @@ class TestPackageInstallUpdateUpgrade(base.CloudTestCase): self.assertIn( 'Commandline: /usr/bin/apt-get --option=Dpkg::Options' '::=--force-confold --option=Dpkg::options::=--force-unsafe-io ' - '--assume-yes --quiet install htop tree', out) + '--assume-yes --quiet install sl tree', out) def test_cloud_init_output(self): """Test cloud-init-output for install & upgrade stuff.""" out = self.get_data_file('cloud-init-output.log') self.assertIn('Setting up tree (', out) - self.assertIn('Setting up htop (', out) + self.assertIn('Setting up sl (', out) self.assertIn('Reading package lists...', out) self.assertIn('Building dependency tree...', out) self.assertIn('Reading state information...', out) diff --git a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml index 71d24b83..dd79e438 100644 --- a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml +++ b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml @@ -15,7 +15,7 @@ required_features: cloud_config: | #cloud-config packages: - - htop + - sl - tree package_update: true package_upgrade: true @@ -23,11 +23,8 @@ collect_scripts: apt_history_cmdline: | #!/bin/bash grep ^Commandline: /var/log/apt/history.log - dpkg_htop: | + dpkg_show: | #!/bin/bash - dpkg -l | grep htop | wc -l - dpkg_tree: | - #!/bin/bash - dpkg -l | grep tree | wc -l + dpkg-query --show # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/salt_minion.py b/tests/cloud_tests/testcases/modules/salt_minion.py index 70917a4c..fc9688ed 100644 --- a/tests/cloud_tests/testcases/modules/salt_minion.py +++ b/tests/cloud_tests/testcases/modules/salt_minion.py @@ -33,7 +33,6 @@ class Test(base.CloudTestCase): def test_minion_installed(self): """Test if the salt-minion package is installed""" - out = self.get_data_file('minion_installed') - self.assertEqual(1, int(out)) + self.assertPackageInstalled('salt-minion') # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/salt_minion.yaml b/tests/cloud_tests/testcases/modules/salt_minion.yaml index f20b9765..9227147c 100644 --- a/tests/cloud_tests/testcases/modules/salt_minion.yaml +++ b/tests/cloud_tests/testcases/modules/salt_minion.yaml @@ -28,15 +28,22 @@ collect_scripts: cat /etc/salt/minion_id minion.pem: | #!/bin/bash - cat /etc/salt/pki/minion/minion.pem + PRIV_KEYFILE=/etc/salt/pki/minion/minion.pem + if [ ! -f $PRIV_KEYFILE ]; then + # Bionic and later automatically moves /etc/salt/pki/minion/* + PRIV_KEYFILE=/var/lib/salt/pki/minion/minion.pem + fi + cat $PRIV_KEYFILE minion.pub: | #!/bin/bash - cat /etc/salt/pki/minion/minion.pub + PUB_KEYFILE=/etc/salt/pki/minion/minion.pub + if [ ! -f $PUB_KEYFILE ]; then + # Bionic and later automatically moves /etc/salt/pki/minion/* + PUB_KEYFILE=/var/lib/salt/pki/minion/minion.pub + fi + cat $PUB_KEYFILE grains: | #!/bin/bash cat /etc/salt/grains - minion_installed: | - #!/bin/bash - dpkg -l | grep salt-minion | grep ii | wc -l # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/user_groups.py b/tests/cloud_tests/testcases/modules/user_groups.py index 93b7a82d..4067348d 100644 --- a/tests/cloud_tests/testcases/modules/user_groups.py +++ b/tests/cloud_tests/testcases/modules/user_groups.py @@ -42,7 +42,7 @@ class TestUserGroups(base.CloudTestCase): def test_user_root_in_secret(self): """Test root user is in 'secret' group.""" - user, _, groups = self.get_data_file('root_groups').partition(":") + _user, _, groups = self.get_data_file('root_groups').partition(":") self.assertIn("secret", groups.split(), msg="User root is not in group 'secret'") diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py index 3dd4996d..06f7d865 100644 --- a/tests/cloud_tests/util.py +++ b/tests/cloud_tests/util.py @@ -358,7 +358,7 @@ class TargetBase(object): # when sh is invoked with '-c', then the first argument is "$0" # which is commonly understood as the "program name". # 'read_data' is the program name, and 'remote_path' is '$1' - stdout, stderr, rc = self._execute( + stdout, _stderr, rc = self._execute( ["sh", "-c", 'exec cat "$1"', 'read_data', remote_path]) if rc != 0: raise RuntimeError("Failed to read file '%s'" % remote_path) diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py index 5a68a484..bfb27444 100644 --- a/tests/cloud_tests/verify.py +++ b/tests/cloud_tests/verify.py @@ -56,6 +56,51 @@ def verify_data(data_dir, platform, os_name, tests): return res +def format_test_failures(test_result): + """Return a human-readable printable format of test failures.""" + if not test_result['failures']: + return '' + failure_hdr = ' test failures:' + failure_fmt = ' * {module}.{class}.{function}\n {error}' + output = [] + for failure in test_result['failures']: + if not output: + output = [failure_hdr] + output.append(failure_fmt.format(**failure)) + return '\n'.join(output) + + +def format_results(res): + """Return human-readable results as a string""" + platform_hdr = 'Platform: {platform}' + distro_hdr = ' Distro: {distro}' + distro_summary_fmt = ( + ' test modules passed:{passed} tests failed:{failed}') + output = [''] + counts = {} + for platform, platform_data in res.items(): + output.append(platform_hdr.format(platform=platform)) + counts[platform] = {} + for distro, distro_data in platform_data.items(): + distro_failure_output = [] + output.append(distro_hdr.format(distro=distro)) + counts[platform][distro] = {'passed': 0, 'failed': 0} + for _, test_result in distro_data.items(): + if test_result['passed']: + counts[platform][distro]['passed'] += 1 + else: + counts[platform][distro]['failed'] += len( + test_result['failures']) + failure_output = format_test_failures(test_result) + if failure_output: + distro_failure_output.append(failure_output) + output.append( + distro_summary_fmt.format(**counts[platform][distro])) + if distro_failure_output: + output.extend(distro_failure_output) + return '\n'.join(output) + + def verify(args): """Verify test data. @@ -90,7 +135,7 @@ def verify(args): failed += len(fail_list) # dump results - LOG.debug('verify results: %s', res) + LOG.debug('\n---- Verify summarized results:\n%s', format_results(res)) if args.result: util.merge_results({'verify': res}, args.result) |