Integration Testing: tox env, pyxld 2.2.3, and revamp framework

Massive update to clean up and greatly enhance the integration testing
framework developed by Wesley Wiedenmeier.

 - Updated tox environment to run integration test 'citest' to utilize
   pylxd 2.2.3
 - Add support for distro feature flags
   - add framework for feature flags to release config with feature groups
     and overrides allowed in any release conf override level
   - add support for feature flags in platform and config handling
   - during collect, skip testcases that require features not supported by
     the image with a warning message
 - Enable additional distros (i.e. centos, debian)
 - Add 'bddeb' command to build a deb from the current working tree
   cleanly in a container, so deps do not have to be installed on host
 - Adds a command line option '--preserve-data' that ensures that
   collected data will be left after tests run. This also allows the
   directory to store collected data in during the run command to be
   specified using '--data-dir'.
 - Updated Read the Docs testing page and doc strings for pep 257
   compliance

1 files changed, 95 insertions, 67 deletions

diff --git a/tests/cloud_tests/instances/base.py b/tests/cloud_tests/instances/base.py
index 9559d286..959e9cce 100644
--- a/tests/cloud_tests/instances/base.py
+++ b/tests/cloud_tests/instances/base.py
@@ -1,120 +1,148 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-import os
-import uuid
+"""Base instance."""
 
 
 class Instance(object):
-    """
-    Base instance object
-    """
+    """Base instance object."""
+
     platform_name = None
 
-    def __init__(self, name):
-        """
-        setup
+    def __init__(self, platform, name, properties, config, features):
+        """Set up instance.
+
+        @param platform: platform object
+        @param name: hostname of instance
+        @param properties: image properties
+        @param config: image config
+        @param features: supported feature flags
         """
+        self.platform = platform
         self.name = name
+        self.properties = properties
+        self.config = config
+        self.features = features
 
-    def execute(self, command, stdin=None, stdout=None, stderr=None, env={}):
-        """
-        command: the command to execute as root inside the image
-        stdin, stderr, stdout: file handles
-        env: environment variables
+    def execute(self, command, stdout=None, stderr=None, env={},
+                rcs=None, description=None):
+        """Execute command in instance, recording output, error and exit code.
 
-        Execute assumes functional networking and execution as root with the
+        Assumes functional networking and execution as root with the
         target filesystem being available at /.
 
-        return_value: tuple containing stdout data, stderr data, exit code
+        @param command: the command to execute as root inside the image
+        @param stdout, stderr: file handles to write output and error to
+        @param env: environment variables
+        @param rcs: allowed return codes from command
+        @param description: purpose of command
+        @return_value: tuple containing stdout data, stderr data, exit code
         """
         raise NotImplementedError
 
-    def read_data(self, remote_path, encode=False):
-        """
-        read_data from instance filesystem
-        remote_path: path in instance
-        decode: return as string
-        return_value: data as str or bytes
+    def read_data(self, remote_path, decode=False):
+        """Read data from instance filesystem.
+
+        @param remote_path: path in instance
+        @param decode: return as string
+        @return_value: data as str or bytes
         """
         raise NotImplementedError
 
     def write_data(self, remote_path, data):
-        """
-        write data to instance filesystem
-        remote_path: path in instance
-        data: data to write, either str or bytes
+        """Write data to instance filesystem.
+
+        @param remote_path: path in instance
+        @param data: data to write, either str or bytes
         """
         raise NotImplementedError
 
     def pull_file(self, remote_path, local_path):
-        """
-        copy file at 'remote_path', from instance to 'local_path'
+        """Copy file at 'remote_path', from instance to 'local_path'.
+
+        @param remote_path: path on remote instance
+        @param local_path: path on local instance
         """
         with open(local_path, 'wb') as fp:
-            fp.write(self.read_data(remote_path), encode=True)
+            fp.write(self.read_data(remote_path))
 
     def push_file(self, local_path, remote_path):
-        """
-        copy file at 'local_path' to instance at 'remote_path'
+        """Copy file at 'local_path' to instance at 'remote_path'.
+
+        @param local_path: path on local instance
+        @param remote_path: path on remote instance
         """
         with open(local_path, 'rb') as fp:
             self.write_data(remote_path, fp.read())
 
-    def run_script(self, script):
+    def run_script(self, script, rcs=None, description=None):
+        """Run script in target and return stdout.
+
+        @param script: script contents
+        @param rcs: allowed return codes from script
+        @param description: purpose of script
+        @return_value: stdout from script
         """
-        run script in target and return stdout
+        script_path = self.tmpfile()
+        try:
+            self.write_data(script_path, script)
+            return self.execute(
+                ['/bin/bash', script_path], rcs=rcs, description=description)
+        finally:
+            self.execute(['rm', script_path], rcs=rcs)
+
+    def tmpfile(self):
+        """Get a tmp file in the target.
+
+        @return_value: path to new file in target
         """
-        script_path = os.path.join('/tmp', str(uuid.uuid1()))
-        self.write_data(script_path, script)
-        (out, err, exit_code) = self.execute(['/bin/bash', script_path])
-        return out
+        return self.execute(['mktemp'])[0].strip()
 
     def console_log(self):
-        """
-        return_value: bytes of this instance’s console
+        """Instance console.
+
+        @return_value: bytes of this instance’s console
         """
         raise NotImplementedError
 
     def reboot(self, wait=True):
-        """
-        reboot instance
-        """
+        """Reboot instance."""
         raise NotImplementedError
 
     def shutdown(self, wait=True):
-        """
-        shutdown instance
-        """
+        """Shutdown instance."""
         raise NotImplementedError
 
-    def start(self, wait=True):
-        """
-        start instance
-        """
+    def start(self, wait=True, wait_for_cloud_init=False):
+        """Start instance."""
         raise NotImplementedError
 
     def destroy(self):
-        """
-        clean up instance
-        """
+        """Clean up instance."""
         pass
 
-    def _wait_for_cloud_init(self, wait_time):
-        """
-        wait until system has fully booted and cloud-init has finished
+    def _wait_for_system(self, wait_for_cloud_init):
+        """Wait until system has fully booted and cloud-init has finished.
+
+        @param wait_time: maximum time to wait
+        @return_value: None, may raise OSError if wait_time exceeded
         """
-        if not wait_time:
-            return
-
-        found_msg = 'found'
-        cmd = ('for ((i=0;i<{wait};i++)); do [ -f "{file}" ] && '
-               '{{ echo "{msg}";break; }} || sleep 1; done').format(
-            file='/run/cloud-init/result.json',
-            wait=wait_time, msg=found_msg)
-
-        (out, err, exit) = self.execute(['/bin/bash', '-c', cmd])
-        if out.strip() != found_msg:
-            raise OSError('timeout: after {}s, cloud-init has not started'
-                          .format(wait_time))
+        def clean_test(test):
+            """Clean formatting for system ready test testcase."""
+            return ' '.join(l for l in test.strip().splitlines()
+                            if not l.lstrip().startswith('#'))
+
+        time = self.config['boot_timeout']
+        tests = [self.config['system_ready_script']]
+        if wait_for_cloud_init:
+            tests.append(self.config['cloud_init_ready_script'])
+
+        formatted_tests = ' && '.join(clean_test(t) for t in tests)
+        test_cmd = ('for ((i=0;i<{time};i++)); do {test} && exit 0; sleep 1; '
+                    'done; exit 1;').format(time=time, test=formatted_tests)
+        cmd = ['/bin/bash', '-c', test_cmd]
+
+        if self.execute(cmd, rcs=(0, 1))[-1] != 0:
+            raise OSError('timeout: after {}s system not started'.format(time))
+
 
 # vi: ts=4 expandtab