From 47016791ca5e97d80e45d3f100bc4e5d0b88627d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 5 Dec 2017 17:05:29 -0500 Subject: tests: consolidate platforms into specific dirs This groups up each test platform into its own directory rather than having files spread between four different directories for one platform. Platforms tend to be worked on one at a time and so having the platforms together makes more sense than apart. --- tests/cloud_tests/platforms/lxd/instance.py | 157 ++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 tests/cloud_tests/platforms/lxd/instance.py (limited to 'tests/cloud_tests/platforms/lxd/instance.py') diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py new file mode 100644 index 00000000..0d697c05 --- /dev/null +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -0,0 +1,157 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base LXD instance.""" + +import os +import shutil +from tempfile import mkdtemp + +from ..instances import Instance + + +class LXDInstance(Instance): + """LXD container backed instance.""" + + platform_name = "lxd" + + def __init__(self, platform, name, properties, config, features, + pylxd_container): + """Set up instance. + + @param platform: platform object + @param name: hostname of instance + @param properties: image properties + @param config: image config + @param features: supported feature flags + """ + self._pylxd_container = pylxd_container + super(LXDInstance, self).__init__( + platform, name, properties, config, features) + self.tmpd = mkdtemp(prefix="%s-%s" % (type(self).__name__, name)) + self._setup_console_log() + + @property + def pylxd_container(self): + """Property function.""" + self._pylxd_container.sync() + return self._pylxd_container + + def _setup_console_log(self): + logf = os.path.join(self.tmpd, "console.log") + + # doing this ensures we can read it. Otherwise it ends up root:root. + with open(logf, "w") as fp: + fp.write("# %s\n" % self.name) + + cfg = "lxc.console.logfile=%s" % logf + orig = self._pylxd_container.config.get('raw.lxc', "") + if orig: + orig += "\n" + self._pylxd_container.config['raw.lxc'] = orig + cfg + self._pylxd_container.save() + self._console_log_file = logf + + def _execute(self, command, stdin=None, env=None): + if env is None: + env = {} + + if stdin is not None: + # pylxd does not support input to execute. + # https://github.com/lxc/pylxd/issues/244 + # + # The solution here is write a tmp file in the container + # and then execute a shell that sets it standard in to + # be from that file, removes it, and calls the comand. + tmpf = self.tmpfile() + self.write_data(tmpf, stdin) + ncmd = 'exec <"{tmpf}"; rm -f "{tmpf}"; exec "$@"' + command = (['sh', '-c', ncmd.format(tmpf=tmpf), 'stdinhack'] + + list(command)) + + # ensure instance is running and execute the command + self.start() + # execute returns a ContainerExecuteResult, named tuple + # (exit_code, stdout, stderr) + res = self.pylxd_container.execute(command, environment=env) + + # get out, exit and err from pylxd return + if not hasattr(res, 'exit_code'): + # pylxd 2.1.3 and earlier only return out and err, no exit + raise RuntimeError( + "No 'exit_code' in pylxd.container.execute return.\n" + "pylxd > 2.2 is required.") + + return res.stdout, res.stderr, res.exit_code + + def read_data(self, remote_path, decode=False): + """Read data from instance filesystem. + + @param remote_path: path in instance + @param decode: decode data before returning. + @return_value: content of remote_path as bytes if 'decode' is False, + and as string if 'decode' is True. + """ + data = self.pylxd_container.files.get(remote_path) + return data.decode() if decode else data + + def write_data(self, remote_path, data): + """Write data to instance filesystem. + + @param remote_path: path in instance + @param data: data to write in bytes + """ + self.pylxd_container.files.put(remote_path, data) + + def console_log(self): + """Console log. + + @return_value: bytes of this instance’s console + """ + if not os.path.exists(self._console_log_file): + raise NotImplementedError( + "Console log '%s' does not exist. If this is a remote " + "lxc, then this is really NotImplementedError. If it is " + "A local lxc, then this is a RuntimeError." + "https://github.com/lxc/lxd/issues/1129") + with open(self._console_log_file, "rb") as fp: + return fp.read() + + def reboot(self, wait=True): + """Reboot instance.""" + self.shutdown(wait=wait) + self.start(wait=wait) + + def shutdown(self, wait=True): + """Shutdown instance.""" + if self.pylxd_container.status != 'Stopped': + self.pylxd_container.stop(wait=wait) + + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance.""" + if self.pylxd_container.status != 'Running': + self.pylxd_container.start(wait=wait) + if wait: + self._wait_for_system(wait_for_cloud_init) + + def freeze(self): + """Freeze instance.""" + if self.pylxd_container.status != 'Frozen': + self.pylxd_container.freeze(wait=True) + + def unfreeze(self): + """Unfreeze instance.""" + if self.pylxd_container.status == 'Frozen': + self.pylxd_container.unfreeze(wait=True) + + def destroy(self): + """Clean up instance.""" + self.unfreeze() + self.shutdown() + self.pylxd_container.delete(wait=True) + if self.platform.container_exists(self.name): + raise OSError('container {} was not properly removed' + .format(self.name)) + shutil.rmtree(self.tmpd) + super(LXDInstance, self).destroy() + +# vi: ts=4 expandtab -- cgit v1.2.3 From bc84f5023f795c261e32cf0690b2d29e12cfaedd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 25 Jan 2018 12:26:34 -0700 Subject: tests: Collect script output as binary, collect systemd journal, fix lxd. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds collection a gzip compressed systemd journal on systemd systems. The file can later be reviewed with:   zcat system.journal.gz > system.journal   journalctl --file=system.journal [-o short-monotonic ..] To support this:   * modify test harness infrastructure to not assume content is utf-8.   * fix lxd platform to support make '_execute' return bytes rather     than a string. https://github.com/lxc/pylxd/issues/268 Also switched the base collectors to use /bin/sh as others already did. --- tests/cloud_tests/collect.py | 7 +++++ tests/cloud_tests/platforms/lxd/instance.py | 43 +++++++++++++---------------- tests/cloud_tests/testcases.yaml | 27 ++++++++++++++---- tests/cloud_tests/testcases/base.py | 6 ++-- tests/cloud_tests/verify.py | 2 +- 5 files changed, 52 insertions(+), 33 deletions(-) (limited to 'tests/cloud_tests/platforms/lxd/instance.py') diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index 33acbb1e..5ea88e50 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -24,6 +24,13 @@ def collect_script(instance, base_dir, script, script_name): (out, err, exit) = instance.run_script( script.encode(), rcs=False, description='collect: {}'.format(script_name)) + if err: + LOG.debug("collect script %s had stderr: %s", script_name, err) + if not isinstance(out, bytes): + raise util.PlatformError( + "Collection of '%s' returned type %s, expected bytes: %s" % + (script_name, type(out), out)) + c_util.write_file(os.path.join(base_dir, script_name), out) diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py index 0d697c05..d2d2a1fd 100644 --- a/tests/cloud_tests/platforms/lxd/instance.py +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -6,6 +6,8 @@ import os import shutil from tempfile import mkdtemp +from cloudinit.util import subp, ProcessExecutionError + from ..instances import Instance @@ -29,6 +31,7 @@ class LXDInstance(Instance): platform, name, properties, config, features) self.tmpd = mkdtemp(prefix="%s-%s" % (type(self).__name__, name)) self._setup_console_log() + self.name = name @property def pylxd_container(self): @@ -55,33 +58,25 @@ class LXDInstance(Instance): if env is None: env = {} - if stdin is not None: - # pylxd does not support input to execute. - # https://github.com/lxc/pylxd/issues/244 - # - # The solution here is write a tmp file in the container - # and then execute a shell that sets it standard in to - # be from that file, removes it, and calls the comand. - tmpf = self.tmpfile() - self.write_data(tmpf, stdin) - ncmd = 'exec <"{tmpf}"; rm -f "{tmpf}"; exec "$@"' - command = (['sh', '-c', ncmd.format(tmpf=tmpf), 'stdinhack'] + - list(command)) + env_args = [] + if env: + env_args = ['env'] + ["%s=%s" for k, v in env.items()] # ensure instance is running and execute the command self.start() - # execute returns a ContainerExecuteResult, named tuple - # (exit_code, stdout, stderr) - res = self.pylxd_container.execute(command, environment=env) - - # get out, exit and err from pylxd return - if not hasattr(res, 'exit_code'): - # pylxd 2.1.3 and earlier only return out and err, no exit - raise RuntimeError( - "No 'exit_code' in pylxd.container.execute return.\n" - "pylxd > 2.2 is required.") - - return res.stdout, res.stderr, res.exit_code + + # Use cmdline client due to https://github.com/lxc/pylxd/issues/268 + exit_code = 0 + try: + stdout, stderr = subp( + ['lxc', 'exec', self.name, '--'] + env_args + list(command), + data=stdin, decode=False) + except ProcessExecutionError as e: + exit_code = e.exit_code + stdout = e.stdout + stderr = e.stderr + + return stdout, stderr, exit_code def read_data(self, remote_path, decode=False): """Read data from instance filesystem. diff --git a/tests/cloud_tests/testcases.yaml b/tests/cloud_tests/testcases.yaml index 7183e017..8e0fb62f 100644 --- a/tests/cloud_tests/testcases.yaml +++ b/tests/cloud_tests/testcases.yaml @@ -7,22 +7,37 @@ base_test_data: #cloud-config collect_scripts: cloud-init.log: | - #!/bin/bash + #!/bin/sh cat /var/log/cloud-init.log cloud-init-output.log: | - #!/bin/bash + #!/bin/sh cat /var/log/cloud-init-output.log instance-id: | - #!/bin/bash + #!/bin/sh cat /run/cloud-init/.instance-id result.json: | - #!/bin/bash + #!/bin/sh cat /run/cloud-init/result.json status.json: | - #!/bin/bash + #!/bin/sh cat /run/cloud-init/status.json cloud-init-version: | - #!/bin/bash + #!/bin/sh dpkg-query -W -f='${Version}' cloud-init + system.journal.gz: | + #!/bin/sh + [ -d /run/systemd ] || { echo "not systemd."; exit 0; } + fail() { echo "ERROR:" "$@" 1>&2; exit 1; } + journal="" + for d in /run/log/journal /var/log/journal; do + for f in $d/*/system.journal; do + [ -f "$f" ] || continue + [ -z "$journal" ] || + fail "multiple journal found: $f $journal." + journal="$f" + done + done + [ -f "$journal" ] || fail "no journal file found." + gzip --to-stdout "$journal" # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 1c5b5405..20e95955 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -30,12 +30,14 @@ class CloudTestCase(unittest.TestCase): raise AssertionError('Key "{}" not in cloud config'.format(name)) return self.cloud_config[name] - def get_data_file(self, name): + def get_data_file(self, name, decode=True): """Get data file failing test if it is not present.""" if name not in self.data: raise AssertionError('File "{}" missing from collect data' .format(name)) - return self.data[name] + if not decode: + return self.data[name] + return self.data[name].decode('utf-8') def get_instance_id(self): """Get recorded instance id.""" diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py index fc1efcfc..2a9fd520 100644 --- a/tests/cloud_tests/verify.py +++ b/tests/cloud_tests/verify.py @@ -29,7 +29,7 @@ def verify_data(base_dir, tests): data = {} test_dir = os.path.join(base_dir, test_name) for script_name in os.listdir(test_dir): - with open(os.path.join(test_dir, script_name), 'r') as fp: + with open(os.path.join(test_dir, script_name), 'rb') as fp: data[script_name] = fp.read() # get test suite and launch tests -- cgit v1.2.3