Add upgrade integration test (#693)

Add an integration test that roughly mimics many of the manual cloud
SRU tests. Also refactored some of the image setup code to make it
easier to use in non-fixture code.

3 files changed, 175 insertions, 45 deletions

diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py
index cc545b0f..160fc085 100644
--- a/tests/integration_tests/conftest.py
+++ b/tests/integration_tests/conftest.py
@@ -1,8 +1,8 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 import datetime
 import logging
-import os
 import pytest
+import os
 import sys
 from tarfile import TarFile
 from contextlib import contextmanager
@@ -14,11 +14,15 @@ from tests.integration_tests.clouds import (
     Ec2Cloud,
     GceCloud,
     ImageSpecification,
+    IntegrationCloud,
     LxdContainerCloud,
     LxdVmCloud,
     OciCloud,
 )
-from tests.integration_tests.instances import IntegrationInstance
+from tests.integration_tests.instances import (
+    CloudInitSource,
+    IntegrationInstance,
+)
 
 
 log = logging.getLogger('integration_testing')
@@ -95,39 +99,42 @@ def session_cloud():
         cloud.destroy()
 
 
-@pytest.fixture(scope='session', autouse=True)
-def setup_image(session_cloud):
+def get_validated_source(
+    source=integration_settings.CLOUD_INIT_SOURCE
+) -> CloudInitSource:
+    if source == 'NONE':
+        return CloudInitSource.NONE
+    elif source == 'IN_PLACE':
+        if session_cloud.datasource not in ['lxd_container', 'lxd_vm']:
+            raise ValueError(
+                'IN_PLACE as CLOUD_INIT_SOURCE only works for LXD')
+        return CloudInitSource.IN_PLACE
+    elif source == 'PROPOSED':
+        return CloudInitSource.PROPOSED
+    elif source.startswith('ppa:'):
+        return CloudInitSource.PPA
+    elif os.path.isfile(str(source)):
+        return CloudInitSource.DEB_PACKAGE
+    raise ValueError(
+        'Invalid value for CLOUD_INIT_SOURCE setting: {}'.format(source))
+
+
+@pytest.fixture(scope='session')
+def setup_image(session_cloud: IntegrationCloud):
     """Setup the target environment with the correct version of cloud-init.
 
     So we can launch instances / run tests with the correct image
     """
-    client = None
+
+    source = get_validated_source()
+    if not source.installs_new_version():
+        return
     log.info('Setting up environment for %s', session_cloud.datasource)
-    if integration_settings.CLOUD_INIT_SOURCE == 'NONE':
-        pass  # that was easy
-    elif integration_settings.CLOUD_INIT_SOURCE == 'IN_PLACE':
-        if session_cloud.datasource not in ['lxd_container', 'lxd_vm']:
-            raise ValueError(
-                'IN_PLACE as CLOUD_INIT_SOURCE only works for LXD')
-        # The mount needs to happen after the instance is created, so
-        # no further action needed here
-    elif integration_settings.CLOUD_INIT_SOURCE == 'PROPOSED':
-        client = session_cloud.launch()
-        client.install_proposed_image()
-    elif integration_settings.CLOUD_INIT_SOURCE.startswith('ppa:'):
-        client = session_cloud.launch()
-        client.install_ppa(integration_settings.CLOUD_INIT_SOURCE)
-    elif os.path.isfile(str(integration_settings.CLOUD_INIT_SOURCE)):
-        client = session_cloud.launch()
-        client.install_deb()
-    else:
-        raise ValueError(
-            'Invalid value for CLOUD_INIT_SOURCE setting: {}'.format(
-                integration_settings.CLOUD_INIT_SOURCE))
-    if client:
-        # Even if we're keeping instances, we don't want to keep this
-        # one around as it was just for image creation
-        client.destroy()
+    client = session_cloud.launch()
+    client.install_new_cloud_init(source)
+    # Even if we're keeping instances, we don't want to keep this
+    # one around as it was just for image creation
+    client.destroy()
     log.info('Done with environment setup')
 
 
@@ -193,21 +200,21 @@ def _client(request, fixture_utils, session_cloud):
 
 
 @pytest.yield_fixture
-def client(request, fixture_utils, session_cloud):
+def client(request, fixture_utils, session_cloud, setup_image):
     """Provide a client that runs for every test."""
     with _client(request, fixture_utils, session_cloud) as client:
         yield client
 
 
 @pytest.yield_fixture(scope='module')
-def module_client(request, fixture_utils, session_cloud):
+def module_client(request, fixture_utils, session_cloud, setup_image):
     """Provide a client that runs once per module."""
     with _client(request, fixture_utils, session_cloud) as client:
         yield client
 
 
 @pytest.yield_fixture(scope='class')
-def class_client(request, fixture_utils, session_cloud):
+def class_client(request, fixture_utils, session_cloud, setup_image):
     """Provide a client that runs once per class."""
     with _client(request, fixture_utils, session_cloud) as client:
         yield client
diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py
index f8f98e42..4321ce07 100644
--- a/tests/integration_tests/instances.py
+++ b/tests/integration_tests/instances.py
@@ -1,4 +1,5 @@
 # This file is part of cloud-init. See LICENSE file for license information.
+from enum import Enum
 import logging
 import os
 import uuid
@@ -25,6 +26,26 @@ def _get_tmp_path():
     return '/var/tmp/{}.tmp'.format(tmp_filename)
 
 
+class CloudInitSource(Enum):
+    """Represents the cloud-init image source setting as a defined value.
+
+    Values here represent all possible values for CLOUD_INIT_SOURCE in
+    tests/integration_tests/integration_settings.py. See that file for an
+    explanation of these values. If the value set there can't be parsed into
+    one of these values, an exception will be raised
+    """
+    NONE = 1
+    IN_PLACE = 2
+    PROPOSED = 3
+    PPA = 4
+    DEB_PACKAGE = 5
+
+    def installs_new_version(self):
+        if self.name in [self.NONE.name, self.IN_PLACE.name]:
+            return False
+        return True
+
+
 class IntegrationInstance:
     def __init__(self, cloud: 'IntegrationCloud', instance: BaseInstance,
                  settings=integration_settings):
@@ -92,16 +113,32 @@ class IntegrationInstance:
             os.unlink(tmp_file.name)
 
     def snapshot(self):
-        return self.cloud.snapshot(self.instance)
-
-    def _install_new_cloud_init(self, remote_script):
-        self.execute(remote_script)
+        image_id = self.cloud.snapshot(self.instance)
+        log.info('Created new image: %s', image_id)
+        return image_id
+
+    def install_new_cloud_init(
+        self,
+        source: CloudInitSource,
+        take_snapshot=True
+    ):
+        if source == CloudInitSource.DEB_PACKAGE:
+            self.install_deb()
+        elif source == CloudInitSource.PPA:
+            self.install_ppa()
+        elif source == CloudInitSource.PROPOSED:
+            self.install_proposed_image()
+        else:
+            raise Exception(
+                "Specified to install {} which isn't supported here".format(
+                    source)
+            )
         version = self.execute('cloud-init -v').split()[-1]
         log.info('Installed cloud-init version: %s', version)
         self.instance.clean()
-        snapshot_id = self.snapshot()
-        log.info('Created new image: %s', snapshot_id)
-        self.cloud.snapshot_id = snapshot_id
+        if take_snapshot:
+            snapshot_id = self.snapshot()
+            self.cloud.snapshot_id = snapshot_id
 
     def install_proposed_image(self):
         log.info('Installing proposed image')
@@ -112,16 +149,16 @@ class IntegrationInstance:
             'apt-get update -q\n'
             'apt-get install -qy cloud-init'
         )
-        self._install_new_cloud_init(remote_script)
+        self.execute(remote_script)
 
-    def install_ppa(self, repo):
+    def install_ppa(self):
         log.info('Installing PPA')
         remote_script = (
             'add-apt-repository {repo} -y && '
             'apt-get update -q && '
             'apt-get install -qy cloud-init'
-        ).format(repo=repo)
-        self._install_new_cloud_init(remote_script)
+        ).format(repo=self.settings.CLOUD_INIT_SOURCE)
+        self.execute(remote_script)
 
     def install_deb(self):
         log.info('Installing deb package')
@@ -132,7 +169,7 @@ class IntegrationInstance:
             local_path=integration_settings.CLOUD_INIT_SOURCE,
             remote_path=remote_path)
         remote_script = 'dpkg -i {path}'.format(path=remote_path)
-        self._install_new_cloud_init(remote_script)
+        self.execute(remote_script)
 
     def __enter__(self):
         return self
diff --git a/tests/integration_tests/test_upgrade.py b/tests/integration_tests/test_upgrade.py
new file mode 100644
index 00000000..78fbc992
--- /dev/null
+++ b/tests/integration_tests/test_upgrade.py
@@ -0,0 +1,86 @@
+import logging
+import pytest
+from pathlib import Path
+
+from tests.integration_tests.clouds import ImageSpecification, IntegrationCloud
+from tests.integration_tests.conftest import (
+    get_validated_source,
+    session_start_time,
+)
+
+log = logging.getLogger('integration_testing')
+
+USER_DATA = """\
+#cloud-config
+hostname: SRU-worked
+"""
+
+
+def _output_to_compare(instance, file_path, netcfg_path):
+    commands = [
+        'hostname',
+        'dpkg-query --show cloud-init',
+        'cat /run/cloud-init/result.json',
+        # 'cloud-init init' helps us understand if our pickling upgrade paths
+        # have broken across re-constitution of a cached datasource. Some
+        # platforms invalidate their datasource cache on reboot, so we run
+        # it here to ensure we get a dirty run.
+        'cloud-init init'
+        'grep Trace /var/log/cloud-init.log',
+        'cloud-id'
+        'cat {}'.format(netcfg_path),
+        'systemd-analyze',
+        'systemd-analyze blame',
+        'cloud-init analyze show',
+        'cloud-init analyze blame',
+    ]
+    with file_path.open('w') as f:
+        for command in commands:
+            f.write('===== {} ====='.format(command) + '\n')
+            f.write(instance.execute(command) + '\n')
+
+
+@pytest.mark.sru_2020_11
+def test_upgrade(session_cloud: IntegrationCloud):
+    source = get_validated_source()
+    if not source.installs_new_version():
+        pytest.skip("Install method '{}' not supported for this test".format(
+            source
+        ))
+        return  # type checking doesn't understand that skip raises
+
+    launch_kwargs = {
+        'name': 'integration-upgrade-test',
+        'image_id': session_cloud._get_initial_image(),
+        'wait': True,
+    }
+
+    image = ImageSpecification.from_os_image()
+
+    # Get the paths to write test logs
+    output_dir = Path(session_cloud.settings.LOCAL_LOG_PATH)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    base_filename = 'test_upgrade_{os}_{{stage}}_{time}.log'.format(
+        os=image.release,
+        time=session_start_time,
+    )
+    before_path = output_dir / base_filename.format(stage='before')
+    after_path = output_dir / base_filename.format(stage='after')
+
+    # Get the network cfg file
+    netcfg_path = '/dev/null'
+    if image.os == 'ubuntu':
+        netcfg_path = '/etc/netplan/50-cloud-init.yaml'
+        if image.release == 'xenial':
+            netcfg_path = '/etc/network/interfaces.d/50-cloud-init.cfg'
+
+    with session_cloud.launch(
+        launch_kwargs=launch_kwargs, user_data=USER_DATA
+    ) as instance:
+        _output_to_compare(instance, before_path, netcfg_path)
+        instance.install_new_cloud_init(source, take_snapshot=False)
+        instance.execute('hostname something-else')
+        instance.restart(raise_on_cloudinit_failure=True)
+        _output_to_compare(instance, after_path, netcfg_path)
+
+    log.info('Wrote upgrade test logs to %s and %s', before_path, after_path)