Integration Testing: tox env, pyxld 2.2.3, and revamp framework

Massive update to clean up and greatly enhance the integration testing
framework developed by Wesley Wiedenmeier.

 - Updated tox environment to run integration test 'citest' to utilize
   pylxd 2.2.3
 - Add support for distro feature flags
   - add framework for feature flags to release config with feature groups
     and overrides allowed in any release conf override level
   - add support for feature flags in platform and config handling
   - during collect, skip testcases that require features not supported by
     the image with a warning message
 - Enable additional distros (i.e. centos, debian)
 - Add 'bddeb' command to build a deb from the current working tree
   cleanly in a container, so deps do not have to be installed on host
 - Adds a command line option '--preserve-data' that ensures that
   collected data will be left after tests run. This also allows the
   directory to store collected data in during the run command to be
   specified using '--data-dir'.
 - Updated Read the Docs testing page and doc strings for pep 257
   compliance

3 files changed, 174 insertions, 126 deletions

diff --git a/tests/cloud_tests/instances/__init__.py b/tests/cloud_tests/instances/__init__.py
index 85bea99f..fc2e9cbc 100644
--- a/tests/cloud_tests/instances/__init__.py
+++ b/tests/cloud_tests/instances/__init__.py
@@ -1,10 +1,10 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
+"""Main init."""
+
 
 def get_instance(snapshot, *args, **kwargs):
-    """
-    get instance from snapshot
-    """
+    """Get instance from snapshot."""
     return snapshot.launch(*args, **kwargs)
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/instances/base.py b/tests/cloud_tests/instances/base.py
index 9559d286..959e9cce 100644
--- a/tests/cloud_tests/instances/base.py
+++ b/tests/cloud_tests/instances/base.py
@@ -1,120 +1,148 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
-import os
-import uuid
+"""Base instance."""
 
 
 class Instance(object):
-    """
-    Base instance object
-    """
+    """Base instance object."""
+
     platform_name = None
 
-    def __init__(self, name):
-        """
-        setup
+    def __init__(self, platform, name, properties, config, features):
+        """Set up instance.
+
+        @param platform: platform object
+        @param name: hostname of instance
+        @param properties: image properties
+        @param config: image config
+        @param features: supported feature flags
         """
+        self.platform = platform
         self.name = name
+        self.properties = properties
+        self.config = config
+        self.features = features
 
-    def execute(self, command, stdin=None, stdout=None, stderr=None, env={}):
-        """
-        command: the command to execute as root inside the image
-        stdin, stderr, stdout: file handles
-        env: environment variables
+    def execute(self, command, stdout=None, stderr=None, env={},
+                rcs=None, description=None):
+        """Execute command in instance, recording output, error and exit code.
 
-        Execute assumes functional networking and execution as root with the
+        Assumes functional networking and execution as root with the
         target filesystem being available at /.
 
-        return_value: tuple containing stdout data, stderr data, exit code
+        @param command: the command to execute as root inside the image
+        @param stdout, stderr: file handles to write output and error to
+        @param env: environment variables
+        @param rcs: allowed return codes from command
+        @param description: purpose of command
+        @return_value: tuple containing stdout data, stderr data, exit code
         """
         raise NotImplementedError
 
-    def read_data(self, remote_path, encode=False):
-        """
-        read_data from instance filesystem
-        remote_path: path in instance
-        decode: return as string
-        return_value: data as str or bytes
+    def read_data(self, remote_path, decode=False):
+        """Read data from instance filesystem.
+
+        @param remote_path: path in instance
+        @param decode: return as string
+        @return_value: data as str or bytes
         """
         raise NotImplementedError
 
     def write_data(self, remote_path, data):
-        """
-        write data to instance filesystem
-        remote_path: path in instance
-        data: data to write, either str or bytes
+        """Write data to instance filesystem.
+
+        @param remote_path: path in instance
+        @param data: data to write, either str or bytes
         """
         raise NotImplementedError
 
     def pull_file(self, remote_path, local_path):
-        """
-        copy file at 'remote_path', from instance to 'local_path'
+        """Copy file at 'remote_path', from instance to 'local_path'.
+
+        @param remote_path: path on remote instance
+        @param local_path: path on local instance
         """
         with open(local_path, 'wb') as fp:
-            fp.write(self.read_data(remote_path), encode=True)
+            fp.write(self.read_data(remote_path))
 
     def push_file(self, local_path, remote_path):
-        """
-        copy file at 'local_path' to instance at 'remote_path'
+        """Copy file at 'local_path' to instance at 'remote_path'.
+
+        @param local_path: path on local instance
+        @param remote_path: path on remote instance
         """
         with open(local_path, 'rb') as fp:
             self.write_data(remote_path, fp.read())
 
-    def run_script(self, script):
+    def run_script(self, script, rcs=None, description=None):
+        """Run script in target and return stdout.
+
+        @param script: script contents
+        @param rcs: allowed return codes from script
+        @param description: purpose of script
+        @return_value: stdout from script
         """
-        run script in target and return stdout
+        script_path = self.tmpfile()
+        try:
+            self.write_data(script_path, script)
+            return self.execute(
+                ['/bin/bash', script_path], rcs=rcs, description=description)
+        finally:
+            self.execute(['rm', script_path], rcs=rcs)
+
+    def tmpfile(self):
+        """Get a tmp file in the target.
+
+        @return_value: path to new file in target
         """
-        script_path = os.path.join('/tmp', str(uuid.uuid1()))
-        self.write_data(script_path, script)
-        (out, err, exit_code) = self.execute(['/bin/bash', script_path])
-        return out
+        return self.execute(['mktemp'])[0].strip()
 
     def console_log(self):
-        """
-        return_value: bytes of this instance’s console
+        """Instance console.
+
+        @return_value: bytes of this instance’s console
         """
         raise NotImplementedError
 
     def reboot(self, wait=True):
-        """
-        reboot instance
-        """
+        """Reboot instance."""
         raise NotImplementedError
 
     def shutdown(self, wait=True):
-        """
-        shutdown instance
-        """
+        """Shutdown instance."""
         raise NotImplementedError
 
-    def start(self, wait=True):
-        """
-        start instance
-        """
+    def start(self, wait=True, wait_for_cloud_init=False):
+        """Start instance."""
         raise NotImplementedError
 
     def destroy(self):
-        """
-        clean up instance
-        """
+        """Clean up instance."""
         pass
 
-    def _wait_for_cloud_init(self, wait_time):
-        """
-        wait until system has fully booted and cloud-init has finished
+    def _wait_for_system(self, wait_for_cloud_init):
+        """Wait until system has fully booted and cloud-init has finished.
+
+        @param wait_time: maximum time to wait
+        @return_value: None, may raise OSError if wait_time exceeded
         """
-        if not wait_time:
-            return
-
-        found_msg = 'found'
-        cmd = ('for ((i=0;i<{wait};i++)); do [ -f "{file}" ] && '
-               '{{ echo "{msg}";break; }} || sleep 1; done').format(
-            file='/run/cloud-init/result.json',
-            wait=wait_time, msg=found_msg)
-
-        (out, err, exit) = self.execute(['/bin/bash', '-c', cmd])
-        if out.strip() != found_msg:
-            raise OSError('timeout: after {}s, cloud-init has not started'
-                          .format(wait_time))
+        def clean_test(test):
+            """Clean formatting for system ready test testcase."""
+            return ' '.join(l for l in test.strip().splitlines()
+                            if not l.lstrip().startswith('#'))
+
+        time = self.config['boot_timeout']
+        tests = [self.config['system_ready_script']]
+        if wait_for_cloud_init:
+            tests.append(self.config['cloud_init_ready_script'])
+
+        formatted_tests = ' && '.join(clean_test(t) for t in tests)
+        test_cmd = ('for ((i=0;i<{time};i++)); do {test} && exit 0; sleep 1; '
+                    'done; exit 1;').format(time=time, test=formatted_tests)
+        cmd = ['/bin/bash', '-c', test_cmd]
+
+        if self.execute(cmd, rcs=(0, 1))[-1] != 0:
+            raise OSError('timeout: after {}s system not started'.format(time))
+
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/instances/lxd.py b/tests/cloud_tests/instances/lxd.py
index f0aa1214..b9c2cc6b 100644
--- a/tests/cloud_tests/instances/lxd.py
+++ b/tests/cloud_tests/instances/lxd.py
@@ -1,115 +1,135 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
+"""Base LXD instance."""
+
 from tests.cloud_tests.instances import base
+from tests.cloud_tests import util
 
 
 class LXDInstance(base.Instance):
-    """
-    LXD container backed instance
-    """
+    """LXD container backed instance."""
+
     platform_name = "lxd"
 
-    def __init__(self, name, platform, pylxd_container):
-        """
-        setup
+    def __init__(self, platform, name, properties, config, features,
+                 pylxd_container):
+        """Set up instance.
+
+        @param platform: platform object
+        @param name: hostname of instance
+        @param properties: image properties
+        @param config: image config
+        @param features: supported feature flags
         """
-        self.platform = platform
         self._pylxd_container = pylxd_container
-        super(LXDInstance, self).__init__(name)
+        super(LXDInstance, self).__init__(
+            platform, name, properties, config, features)
 
     @property
     def pylxd_container(self):
+        """Property function."""
         self._pylxd_container.sync()
         return self._pylxd_container
 
-    def execute(self, command, stdin=None, stdout=None, stderr=None, env={}):
-        """
-        command: the command to execute as root inside the image
-        stdin, stderr, stdout: file handles
-        env: environment variables
+    def execute(self, command, stdout=None, stderr=None, env={},
+                rcs=None, description=None):
+        """Execute command in instance, recording output, error and exit code.
 
-        Execute assumes functional networking and execution as root with the
+        Assumes functional networking and execution as root with the
         target filesystem being available at /.
 
-        return_value: tuple containing stdout data, stderr data, exit code
+        @param command: the command to execute as root inside the image
+        @param stdout: file handler to write output
+        @param stderr: file handler to write error
+        @param env: environment variables
+        @param rcs: allowed return codes from command
+        @param description: purpose of command
+        @return_value: tuple containing stdout data, stderr data, exit code
         """
-        # TODO: the pylxd api handler for container.execute needs to be
-        #       extended to properly pass in stdin
-        # TODO: the pylxd api handler for container.execute needs to be
-        #       extended to get the return code, for now just use 0
+        # ensure instance is running and execute the command
         self.start()
-        if stdin:
-            raise NotImplementedError
         res = self.pylxd_container.execute(command, environment=env)
-        for (f, data) in (i for i in zip((stdout, stderr), res) if i[0]):
-            f.write(data)
-        return res + (0,)
+
+        # get out, exit and err from pylxd return
+        if hasattr(res, 'exit_code'):
+            # pylxd 2.2 returns ContainerExecuteResult, named tuple of
+            # (exit_code, out, err)
+            (exit, out, err) = res
+        else:
+            # pylxd 2.1.3 and earlier only return out and err, no exit
+            # LOG.warning('using pylxd version < 2.2')
+            (out, err) = res
+            exit = 0
+
+        # write data to file descriptors if needed
+        if stdout:
+            stdout.write(out)
+        if stderr:
+            stderr.write(err)
+
+        # if the command exited with a code not allowed in rcs, then fail
+        if exit not in (rcs if rcs else (0,)):
+            error_desc = ('Failed command to: {}'.format(description)
+                          if description else None)
+            raise util.InTargetExecuteError(
+                out, err, exit, command, self.name, error_desc)
+
+        return (out, err, exit)
 
     def read_data(self, remote_path, decode=False):
-        """
-        read data from instance filesystem
-        remote_path: path in instance
-        decode: return as string
-        return_value: data as str or bytes
+        """Read data from instance filesystem.
+
+        @param remote_path: path in instance
+        @param decode: return as string
+        @return_value: data as str or bytes
         """
         data = self.pylxd_container.files.get(remote_path)
         return data.decode() if decode and isinstance(data, bytes) else data
 
     def write_data(self, remote_path, data):
-        """
-        write data to instance filesystem
-        remote_path: path in instance
-        data: data to write, either str or bytes
+        """Write data to instance filesystem.
+
+        @param remote_path: path in instance
+        @param data: data to write, either str or bytes
         """
         self.pylxd_container.files.put(remote_path, data)
 
     def console_log(self):
-        """
-        return_value: bytes of this instance’s console
+        """Console log.
+
+        @return_value: bytes of this instance’s console
         """
         raise NotImplementedError
 
     def reboot(self, wait=True):
-        """
-        reboot instance
-        """
+        """Reboot instance."""
         self.shutdown(wait=wait)
         self.start(wait=wait)
 
     def shutdown(self, wait=True):
-        """
-        shutdown instance
-        """
+        """Shutdown instance."""
         if self.pylxd_container.status != 'Stopped':
             self.pylxd_container.stop(wait=wait)
 
-    def start(self, wait=True, wait_time=None):
-        """
-        start instance
-        """
+    def start(self, wait=True, wait_for_cloud_init=False):
+        """Start instance."""
         if self.pylxd_container.status != 'Running':
             self.pylxd_container.start(wait=wait)
-            if wait and isinstance(wait_time, int):
-                self._wait_for_cloud_init(wait_time)
+            if wait:
+                self._wait_for_system(wait_for_cloud_init)
 
     def freeze(self):
-        """
-        freeze instance
-        """
+        """Freeze instance."""
         if self.pylxd_container.status != 'Frozen':
             self.pylxd_container.freeze(wait=True)
 
     def unfreeze(self):
-        """
-        unfreeze instance
-        """
+        """Unfreeze instance."""
         if self.pylxd_container.status == 'Frozen':
             self.pylxd_container.unfreeze(wait=True)
 
     def destroy(self):
-        """
-        clean up instance
-        """
+        """Clean up instance."""
         self.unfreeze()
         self.shutdown()
         self.pylxd_container.delete(wait=True)