Collect logs from integration test runs (#675)

During teardown of every cloud instance, run 'cloud-init collect-logs',
then transfer and unpack locally. Two new integration settings have
been added to specify when to perform this action (ALWAYS,
ON_ERROR, NEVER), and where to store these logs.

3 files changed, 60 insertions, 4 deletions

diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py
index 73b44bfc..6e1465be 100644
--- a/tests/integration_tests/conftest.py
+++ b/tests/integration_tests/conftest.py
@@ -1,9 +1,12 @@
 # This file is part of cloud-init. See LICENSE file for license information.
+import datetime
 import logging
 import os
 import pytest
 import sys
+from tarfile import TarFile
 from contextlib import contextmanager
+from pathlib import Path
 
 from tests.integration_tests import integration_settings
 from tests.integration_tests.clouds import (
@@ -14,6 +17,7 @@ from tests.integration_tests.clouds import (
     LxdContainerCloud,
     LxdVmCloud,
 )
+from tests.integration_tests.instances import IntegrationInstance
 
 
 log = logging.getLogger('integration_testing')
@@ -29,6 +33,8 @@ platforms = {
     'lxd_vm': LxdVmCloud,
 }
 
+session_start_time = datetime.datetime.now().strftime('%y%m%d%H%M%S')
+
 
 def pytest_runtest_setup(item):
     """Skip tests on unsupported clouds.
@@ -114,6 +120,43 @@ def setup_image(session_cloud):
     log.info('Done with environment setup')
 
 
+def _collect_logs(instance: IntegrationInstance, node_id: str,
+                  test_failed: bool):
+    """Collect logs from remote instance.
+
+    Args:
+        instance: The current IntegrationInstance to collect logs from
+        node_id: The pytest representation of this test, E.g.:
+            tests/integration_tests/test_example.py::TestExample.test_example
+        test_failed: If test failed or not
+    """
+    if any([
+        integration_settings.COLLECT_LOGS == 'NEVER',
+        integration_settings.COLLECT_LOGS == 'ON_ERROR' and not test_failed
+    ]):
+        return
+    instance.execute(
+        'cloud-init collect-logs -u -t /var/tmp/cloud-init.tar.gz')
+    node_id_path = Path(
+        node_id
+        .replace('.py', '')  # Having a directory with '.py' would be weird
+        .replace('::', os.path.sep)  # Turn classes/tests into paths
+        .replace('[', '-')  # For parametrized names
+        .replace(']', '')  # For parameterized names
+    )
+    log_dir = Path(
+        integration_settings.LOCAL_LOG_PATH
+    ) / session_start_time / node_id_path
+    if not log_dir.exists():
+        log_dir.mkdir(parents=True)
+    tarball_path = log_dir / 'cloud-init.tar.gz'
+    instance.pull_file('/var/tmp/cloud-init.tar.gz', tarball_path)
+
+    tarball = TarFile.open(str(tarball_path))
+    tarball.extractall(path=str(log_dir))
+    tarball_path.unlink()
+
+
 @contextmanager
 def _client(request, fixture_utils, session_cloud):
     """Fixture implementation for the client fixtures.
@@ -132,7 +175,10 @@ def _client(request, fixture_utils, session_cloud):
     with session_cloud.launch(
         user_data=user_data, launch_kwargs=launch_kwargs
     ) as instance:
+        previous_failures = request.session.testsfailed
         yield instance
+        test_failed = request.session.testsfailed - previous_failures > 0
+        _collect_logs(instance, request.node.nodeid, test_failed)
 
 
 @pytest.yield_fixture
diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py
index 9b13288c..a0a5fb6b 100644
--- a/tests/integration_tests/instances.py
+++ b/tests/integration_tests/instances.py
@@ -47,14 +47,14 @@ class IntegrationInstance:
     def pull_file(self, remote_path, local_path):
         # First copy to a temporary directory because of permissions issues
         tmp_path = _get_tmp_path()
-        self.instance.execute('cp {} {}'.format(remote_path, tmp_path))
-        self.instance.pull_file(tmp_path, local_path)
+        self.instance.execute('cp {} {}'.format(str(remote_path), tmp_path))
+        self.instance.pull_file(tmp_path, str(local_path))
 
     def push_file(self, local_path, remote_path):
         # First push to a temporary directory because of permissions issues
         tmp_path = _get_tmp_path()
-        self.instance.push_file(local_path, tmp_path)
-        self.execute('mv {} {}'.format(tmp_path, remote_path))
+        self.instance.push_file(str(local_path), tmp_path)
+        self.execute('mv {} {}'.format(tmp_path, str(remote_path)))
 
     def read_from_file(self, remote_path) -> str:
         result = self.execute('cat {}'.format(remote_path))
diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py
index a0609f7e..9be9a94f 100644
--- a/tests/integration_tests/integration_settings.py
+++ b/tests/integration_tests/integration_settings.py
@@ -55,6 +55,16 @@ EXISTING_INSTANCE_ID = None
 #   A path to a valid package to be uploaded and installed
 CLOUD_INIT_SOURCE = 'NONE'
 
+# Before an instance is torn down, we run `cloud-init collect-logs`
+# and transfer them locally. These settings specify when to collect these
+# logs and where to put them on the local filesystem
+# One of:
+#   'ALWAYS'
+#   'ON_ERROR'
+#   'NEVER'
+COLLECT_LOGS = 'ON_ERROR'
+LOCAL_LOG_PATH = '/tmp/cloud_init_test_logs'
+
 ##################################################################
 # GCE SPECIFIC SETTINGS
 ##################################################################