summaryrefslogtreecommitdiff
path: root/tests/cloud_tests/verify.py
diff options
context:
space:
mode:
Diffstat (limited to 'tests/cloud_tests/verify.py')
-rw-r--r--tests/cloud_tests/verify.py149
1 files changed, 0 insertions, 149 deletions
diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py
deleted file mode 100644
index 0295af40..00000000
--- a/tests/cloud_tests/verify.py
+++ /dev/null
@@ -1,149 +0,0 @@
-# This file is part of cloud-init. See LICENSE file for license information.
-
-"""Verify test results."""
-
-import os
-import unittest
-
-from tests.cloud_tests import (config, LOG, util, testcases)
-
-
-def verify_data(data_dir, platform, os_name, tests):
- """Verify test data is correct.
-
- @param data_dir: top level directory for all tests
- @param platform: The platform name we for this test data (e.g. lxd)
- @param os_name: The operating system under test (xenial, artful, etc.).
- @param tests: list of test names
- @return_value: {<test_name>: {passed: True/False, failures: []}}
- """
- base_dir = os.sep.join((data_dir, platform, os_name))
- runner = unittest.TextTestRunner(verbosity=util.current_verbosity())
- res = {}
- for test_name in tests:
- LOG.debug('verifying test data for %s', test_name)
-
- # get cloudconfig for test
- test_conf = config.load_test_config(test_name)
- test_module = config.name_to_module(test_name)
- cloud_conf = test_conf['cloud_config']
-
- # load script outputs
- data = {'platform': platform, 'os_name': os_name}
- test_dir = os.path.join(base_dir, test_name)
- for script_name in os.listdir(test_dir):
- with open(os.path.join(test_dir, script_name), 'rb') as fp:
- data[script_name] = fp.read()
-
- # get test suite and launch tests
- suite = testcases.get_suite(test_module, data, cloud_conf)
- suite_results = runner.run(suite)
- res[test_name] = {
- 'passed': suite_results.wasSuccessful(),
- 'failures': [{'module': type(test_class).__base__.__module__,
- 'class': type(test_class).__base__.__name__,
- 'function': str(test_class).split()[0],
- 'error': trace.splitlines()[-1],
- 'traceback': trace, }
- for test_class, trace in suite_results.failures]
- }
-
- for failure in res[test_name]['failures']:
- LOG.warning('test case: %s failed %s.%s with: %s',
- test_name, failure['class'], failure['function'],
- failure['error'])
-
- return res
-
-
-def format_test_failures(test_result):
- """Return a human-readable printable format of test failures."""
- if not test_result['failures']:
- return ''
- failure_hdr = ' test failures:'
- failure_fmt = ' * {module}.{class}.{function}\n '
- output = []
- for failure in test_result['failures']:
- if not output:
- output = [failure_hdr]
- msg = failure_fmt.format(**failure)
- if failure.get('error'):
- msg += failure['error']
- else:
- msg += failure.get('traceback', '')
- output.append(msg)
- return '\n'.join(output)
-
-
-def format_results(res):
- """Return human-readable results as a string"""
- platform_hdr = 'Platform: {platform}'
- distro_hdr = ' Distro: {distro}'
- distro_summary_fmt = (
- ' test modules passed:{passed} tests failed:{failed}')
- output = ['']
- counts = {}
- for platform, platform_data in res.items():
- output.append(platform_hdr.format(platform=platform))
- counts[platform] = {}
- for distro, distro_data in platform_data.items():
- distro_failure_output = []
- output.append(distro_hdr.format(distro=distro))
- counts[platform][distro] = {'passed': 0, 'failed': 0}
- for _, test_result in distro_data.items():
- if test_result['passed']:
- counts[platform][distro]['passed'] += 1
- else:
- counts[platform][distro]['failed'] += len(
- test_result['failures'])
- failure_output = format_test_failures(test_result)
- if failure_output:
- distro_failure_output.append(failure_output)
- output.append(
- distro_summary_fmt.format(**counts[platform][distro]))
- if distro_failure_output:
- output.extend(distro_failure_output)
- return '\n'.join(output)
-
-
-def verify(args):
- """Verify test data.
-
- @param args: directory of test data
- @return_value: 0 for success, or number of failed tests
- """
- failed = 0
- res = {}
-
- # find test data
- tests = util.list_test_data(args.data_dir)
-
- for platform in tests.keys():
- res[platform] = {}
- for os_name in tests[platform].keys():
- test_name = "platform='{}', os='{}'".format(platform, os_name)
- LOG.info('test: %s verifying test data', test_name)
-
- # run test
- res[platform][os_name] = verify_data(
- args.data_dir, platform, os_name,
- tests[platform][os_name])
-
- # handle results
- fail_list = [k for k, v in res[platform][os_name].items()
- if not v.get('passed')]
- if len(fail_list) == 0:
- LOG.info('test: %s passed all tests', test_name)
- else:
- LOG.warning('test: %s failed %s tests', test_name,
- len(fail_list))
- failed += len(fail_list)
-
- # dump results
- LOG.debug('\n---- Verify summarized results:\n%s', format_results(res))
- if args.result:
- util.merge_results({'verify': res}, args.result)
-
- return failed
-
-# vi: ts=4 expandtab