From 1081962eacf2814fea6f4fa3255c530de14e4a24 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 19 Apr 2018 21:30:08 -0600 Subject: pylint: pay attention to unused variable warnings. This enables warnings produced by pylint for unused variables (W0612), and fixes the existing errors. --- tests/cloud_tests/collect.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tests/cloud_tests/collect.py') diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index d4f9135b..1ba72856 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -25,7 +25,8 @@ def collect_script(instance, base_dir, script, script_name): script.encode(), rcs=False, description='collect: {}'.format(script_name)) if err: - LOG.debug("collect script %s had stderr: %s", script_name, err) + LOG.debug("collect script %s exited '%s' and had stderr: %s", + script_name, err, exit) if not isinstance(out, bytes): raise util.PlatformError( "Collection of '%s' returned type %s, expected bytes: %s" % -- cgit v1.2.3 From 589b542bfb3b6630b931a506ca017635059cef1d Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Wed, 16 May 2018 08:16:10 -0600 Subject: tests: restructure SSH and initial connections The SSH function was retrying and waiting for SSH for over an hour when an SSH connection was failing to be established. This reduces the amount of retries and time between each retry to prevent tests from running for hours. Also restructures how waiting for the system works: the system will attempt to SSH up to the boot timeout time by catching SSH connection failures and retrying until the timeout is reached. If the limit is reached now an exception is thrown to abort the test. Drive by - this also fixes printing of the instance name when collecting the console log, rather than showing a Python object address. Fixes LP: #1758409 --- tests/cloud_tests/collect.py | 2 +- tests/cloud_tests/platforms/instances.py | 39 ++++++++++++++++++++++++-------- 2 files changed, 30 insertions(+), 11 deletions(-) (limited to 'tests/cloud_tests/collect.py') diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index 1ba72856..78263bf5 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -42,7 +42,7 @@ def collect_console(instance, base_dir): @param base_dir: directory to write console log to """ logfile = os.path.join(base_dir, 'console.log') - LOG.debug('getting console log for %s to %s', instance, logfile) + LOG.debug('getting console log for %s to %s', instance.name, logfile) try: data = instance.console_log() except NotImplementedError as e: diff --git a/tests/cloud_tests/platforms/instances.py b/tests/cloud_tests/platforms/instances.py index cc439d29..95bc3b16 100644 --- a/tests/cloud_tests/platforms/instances.py +++ b/tests/cloud_tests/platforms/instances.py @@ -87,7 +87,12 @@ class Instance(TargetBase): self._ssh_client = None def _ssh_connect(self): - """Connect via SSH.""" + """Connect via SSH. + + Attempt to SSH to the client on the specific IP and port. If it + fails in some manner, then retry 2 more times for a total of 3 + attempts; sleeping a few seconds between attempts. + """ if self._ssh_client: return self._ssh_client @@ -98,21 +103,22 @@ class Instance(TargetBase): client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) private_key = paramiko.RSAKey.from_private_key_file(self.ssh_key_file) - retries = 30 + retries = 3 while retries: try: client.connect(username=self.ssh_username, hostname=self.ssh_ip, port=self.ssh_port, - pkey=private_key, banner_timeout=30) + pkey=private_key) self._ssh_client = client return client except (ConnectionRefusedError, AuthenticationException, BadHostKeyException, ConnectionResetError, SSHException, OSError): retries -= 1 - time.sleep(10) + LOG.debug('Retrying ssh connection on connect failure') + time.sleep(3) - ssh_cmd = 'Failed ssh connection to %s@%s:%s after 300 seconds' % ( + ssh_cmd = 'Failed ssh connection to %s@%s:%s after 3 retries' % ( self.ssh_username, self.ssh_ip, self.ssh_port ) raise util.InTargetExecuteError(b'', b'', 1, ssh_cmd, 'ssh') @@ -128,18 +134,31 @@ class Instance(TargetBase): return ' '.join(l for l in test.strip().splitlines() if not l.lstrip().startswith('#')) - time = self.config['boot_timeout'] + boot_timeout = self.config['boot_timeout'] tests = [self.config['system_ready_script']] if wait_for_cloud_init: tests.append(self.config['cloud_init_ready_script']) formatted_tests = ' && '.join(clean_test(t) for t in tests) cmd = ('i=0; while [ $i -lt {time} ] && i=$(($i+1)); do {test} && ' - 'exit 0; sleep 1; done; exit 1').format(time=time, + 'exit 0; sleep 1; done; exit 1').format(time=boot_timeout, test=formatted_tests) - if self.execute(cmd, rcs=(0, 1))[-1] != 0: - raise OSError('timeout: after {}s system not started'.format(time)) - + end_time = time.time() + boot_timeout + while True: + try: + return_code = self.execute( + cmd, rcs=(0, 1), description='wait for instance start' + )[-1] + if return_code == 0: + break + except util.InTargetExecuteError: + LOG.warning("failed to connect via SSH") + + if time.time() < end_time: + time.sleep(3) + else: + raise util.PlatformError('ssh', 'after %ss instance is not ' + 'reachable' % boot_timeout) # vi: ts=4 expandtab -- cgit v1.2.3 From 7b3c21615dac3b0d9163c9883309a2e7b675622a Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 12 Jun 2018 10:14:07 -0600 Subject: test: add optional --preserve-instance arg to integraiton tests By default, integration tests destroy the test instances after each test run. To aid debug and development of integration tests, support a --preserve-instance argument which will leave the modified test instance in a stopped state for further debug. --- doc/rtd/topics/tests.rst | 7 ++++++- tests/cloud_tests/args.py | 3 +++ tests/cloud_tests/collect.py | 3 ++- tests/cloud_tests/stage.py | 15 ++++++++++++--- 4 files changed, 23 insertions(+), 5 deletions(-) (limited to 'tests/cloud_tests/collect.py') diff --git a/doc/rtd/topics/tests.rst b/doc/rtd/topics/tests.rst index cac4a6e4..b83bd899 100644 --- a/doc/rtd/topics/tests.rst +++ b/doc/rtd/topics/tests.rst @@ -58,7 +58,8 @@ explaining how to run one or the other independently. $ tox -e citest -- run --verbose \ --os-name stretch --os-name xenial \ --deb cloud-init_0.7.8~my_patch_all.deb \ - --preserve-data --data-dir ~/collection + --preserve-data --data-dir ~/collection \ + --preserve-instance The above command will do the following: @@ -76,6 +77,10 @@ The above command will do the following: * ``--preserve-data`` always preserve collected data, do not remove data after successful test run +* ``--preserve-instance`` do not destroy the instance after test to allow + for debugging the stopped instance during integration test development. By + default, test instances are destroyed after the test completes. + * ``--data-dir ~/collection`` write collected data into `~/collection`, rather than using a temporary directory diff --git a/tests/cloud_tests/args.py b/tests/cloud_tests/args.py index c6c1877b..ab345491 100644 --- a/tests/cloud_tests/args.py +++ b/tests/cloud_tests/args.py @@ -62,6 +62,9 @@ ARG_SETS = { (('-d', '--data-dir'), {'help': 'directory to store test data in', 'action': 'store', 'metavar': 'DIR', 'required': False}), + (('--preserve-instance',), + {'help': 'do not destroy the instance under test', + 'action': 'store_true', 'default': False, 'required': False}), (('--preserve-data',), {'help': 'do not remove collected data after successful run', 'action': 'store_true', 'default': False, 'required': False}),), diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index 78263bf5..75b50616 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -93,7 +93,8 @@ def collect_test_data(args, snapshot, os_name, test_name): # create test instance component = PlatformComponent( partial(platforms.get_instance, snapshot, user_data, - block=True, start=False, use_desc=test_name)) + block=True, start=False, use_desc=test_name), + preserve_instance=args.preserve_instance) LOG.info('collecting test data for test: %s', test_name) with component as instance: diff --git a/tests/cloud_tests/stage.py b/tests/cloud_tests/stage.py index 74a7d46d..d64a1dcc 100644 --- a/tests/cloud_tests/stage.py +++ b/tests/cloud_tests/stage.py @@ -12,9 +12,15 @@ from tests.cloud_tests import LOG class PlatformComponent(object): """Context manager to safely handle platform components.""" - def __init__(self, get_func): - """Store get_ function as partial with no args.""" + def __init__(self, get_func, preserve_instance=False): + """Store get_ function as partial with no args. + + @param get_func: Callable returning an instance from the platform. + @param preserve_instance: Boolean, when True, do not destroy instance + after test. Used for test development. + """ self.get_func = get_func + self.preserve_instance = preserve_instance def __enter__(self): """Create instance of platform component.""" @@ -24,7 +30,10 @@ class PlatformComponent(object): def __exit__(self, etype, value, trace): """Destroy instance.""" if self.instance is not None: - self.instance.destroy() + if self.preserve_instance: + LOG.info('Preserving test instance %s', self.instance.name) + else: + self.instance.destroy() def run_single(name, call): -- cgit v1.2.3