Initial implementation of integration testing infrastructure (#581)

10 files changed, 547 insertions, 5 deletions

diff --git a/.gitignore b/.gitignore
index 3589b210..5a68bff9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,3 +27,6 @@ cloud-init_*.dsc
 cloud-init_*.orig.tar.gz
 cloud-init_*.tar.xz
 cloud-init_*.upload
+
+# user test settings
+tests/integration_tests/user_settings.py
diff --git a/HACKING.rst b/HACKING.rst
index 60c7b5e0..4ae7f7b4 100644
--- a/HACKING.rst
+++ b/HACKING.rst
@@ -173,9 +173,18 @@ Cloud Config Modules
 * Any new modules should use underscores in any new config options and not
   hyphens (e.g. `new_option` and *not* `new-option`).
 
-Unit Testing
+.. _unit_testing:
+
+Testing
 ------------
 
+cloud-init has both unit tests and integration tests. Unit tests can
+be found in-tree alongside the source code, as well as
+at ``tests/unittests``. Integration tests can be found at
+``tests/integration_tests``. Documentation specifically for integration
+tests can be found on the :ref:`integration_tests` page, but
+the guidelines specified below apply to both types of tests.
+
 cloud-init uses `pytest`_ to run its tests, and has tests written both
 as ``unittest.TestCase`` sub-classes and as un-subclassed pytest tests.
 The following guidelines should be followed:
diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst
index 0015e35a..ddcb0b31 100644
--- a/doc/rtd/index.rst
+++ b/doc/rtd/index.rst
@@ -75,6 +75,7 @@ Having trouble? We would like to help!
    topics/dir_layout.rst
    topics/analyze.rst
    topics/docs.rst
-   topics/tests.rst
+   topics/integration_tests.rst
+   topics/cloud_tests.rst
 
 .. vi: textwidth=79
diff --git a/doc/rtd/topics/tests.rst b/doc/rtd/topics/cloud_tests.rst
index f03b5969..e4e893d2 100644
--- a/doc/rtd/topics/tests.rst
+++ b/doc/rtd/topics/cloud_tests.rst
@@ -1,6 +1,9 @@
-*******************
-Integration Testing
-*******************
+************************
+Cloud tests (Deprecated)
+************************
+
+Cloud tests are longer be maintained. For writing integration
+tests, see the :ref:`integration_tests` page.
 
 Overview
 ========
diff --git a/doc/rtd/topics/integration_tests.rst b/doc/rtd/topics/integration_tests.rst
new file mode 100644
index 00000000..aeda326c
--- /dev/null
+++ b/doc/rtd/topics/integration_tests.rst
@@ -0,0 +1,81 @@
+.. _integration_tests:
+
+*******************
+Integration Testing
+*******************
+
+Overview
+=========
+
+Integration tests are written using pytest and are located at
+``tests/integration_tests``. General design principles
+laid out in :ref:`unit_testing` should be followed for integration tests.
+
+Setup is accomplished via a set of fixtures located in
+``tests/integration_tests/conftest.py``.
+
+Image Setup
+===========
+
+Image setup occurs once when a test session begins and is implemented
+via fixture. Image setup roughly follows these steps:
+
+* Launch an instance on the specified test platform
+* Install the version of cloud-init under test
+* Run ``cloud-init clean`` on the instance so subsequent boots
+  resemble out of the box behavior
+* Take a snapshot of the instance to be used as a new image from
+  which new instances can be launched
+
+Test Setup
+==============
+Test setup occurs between image setup and test execution. Test setup
+is implemented via one of the ``client`` fixtures. When a client fixture
+is used, a test instance from which to run tests is launched prior to
+test execution and torn down after.
+
+Test Definition
+===============
+Tests are defined like any other pytest test. The ``user_data``
+mark can be used to supply the cloud-config user data. Platform specific
+marks can be used to limit tests to particular platforms. The
+client fixture can be used to interact with the launched
+test instance.
+
+A basic example:
+
+.. code-block:: python
+
+    USER_DATA = """#cloud-config
+    bootcmd:
+    - echo 'hello config!' > /tmp/user_data.txt"""
+
+
+    class TestSimple:
+        @pytest.mark.user_data(USER_DATA)
+        @pytest.mark.ec2
+        def test_simple(self, client):
+            print(client.exec('cloud-init -v'))
+
+Test Execution
+==============
+Test execution happens via pytest. To run all integration tests,
+you would run:
+
+.. code-block:: bash
+
+    pytest tests/integration_tests/
+
+
+Configuration
+=============
+
+All possible configuration values are defined in
+``tests/integration_tests/integration_settings.py``. Defaults can be
+overridden by supplying values in ``tests/integration_tests/user_settings.py``
+or by providing an environment variable of the same name prepended with
+``CLOUD_INIT_``. For example, to set the ``PLATFORM`` setting:
+
+.. code-block:: bash
+
+    CLOUD_INIT_PLATFORM='ec2' pytest tests/integration_tests/
diff --git a/integration-requirements.txt b/integration-requirements.txt
index 13cfb9d7..64455c79 100644
--- a/integration-requirements.txt
+++ b/integration-requirements.txt
@@ -4,6 +4,8 @@
 # Note: Changes to this requirements may require updates to
 # the packages/pkg-deps.json file as well.
 #
+pytest
+git+https://github.com/canonical/pycloudlib.git
 
 # ec2 backend
 boto3==1.14.53
diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py
new file mode 100644
index 00000000..a170bfc9
--- /dev/null
+++ b/tests/integration_tests/conftest.py
@@ -0,0 +1,106 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+import os
+import logging
+import pytest
+import sys
+from contextlib import contextmanager
+
+from tests.integration_tests import integration_settings
+from tests.integration_tests.platforms import (
+    dynamic_client,
+    LxdContainerClient,
+    client_name_to_class
+)
+
+log = logging.getLogger('integration_testing')
+log.addHandler(logging.StreamHandler(sys.stdout))
+log.setLevel(logging.INFO)
+
+
+def pytest_runtest_setup(item):
+    """Skip tests on unsupported clouds.
+
+    A test can take any number of marks to specify the platforms it can
+    run on. If a platform(s) is specified and we're not running on that
+    platform, then skip the test. If platform specific marks are not
+    specified, then we assume the test can be run anywhere.
+    """
+    all_platforms = client_name_to_class.keys()
+    supported_platforms = set(all_platforms).intersection(
+        mark.name for mark in item.iter_markers())
+    current_platform = integration_settings.PLATFORM
+    if supported_platforms and current_platform not in supported_platforms:
+        pytest.skip('Cannot run on platform {}'.format(current_platform))
+
+
+# disable_subp_usage is defined at a higher level, but we don't
+# want it applied here
+@pytest.fixture()
+def disable_subp_usage(request):
+    pass
+
+
+@pytest.fixture(scope='session', autouse=True)
+def setup_image():
+    """Setup the target environment with the correct version of cloud-init.
+
+    So we can launch instances / run tests with the correct image
+    """
+    client = dynamic_client()
+    log.info('Setting up environment for %s', client.datasource)
+    if integration_settings.CLOUD_INIT_SOURCE == 'NONE':
+        pass  # that was easy
+    elif integration_settings.CLOUD_INIT_SOURCE == 'IN_PLACE':
+        if not isinstance(client, LxdContainerClient):
+            raise ValueError(
+                'IN_PLACE as CLOUD_INIT_SOURCE only works for LXD')
+        # The mount needs to happen after the instance is launched, so
+        # no further action needed here
+    elif integration_settings.CLOUD_INIT_SOURCE == 'PROPOSED':
+        client.launch()
+        client.install_proposed_image()
+    elif integration_settings.CLOUD_INIT_SOURCE.startswith('ppa:'):
+        client.launch()
+        client.install_ppa(integration_settings.CLOUD_INIT_SOURCE)
+    elif os.path.isfile(str(integration_settings.CLOUD_INIT_SOURCE)):
+        client.launch()
+        client.install_deb()
+    if client.instance:
+        # Even if we're keeping instances, we don't want to keep this
+        # one around as it was just for image creation
+        client.destroy()
+    log.info('Done with environment setup')
+
+
+@contextmanager
+def _client(request, fixture_utils):
+    """Fixture implementation for the client fixtures.
+
+    Launch the dynamic IntegrationClient instance using any provided
+    userdata, yield to the test, then cleanup
+    """
+    user_data = fixture_utils.closest_marker_first_arg_or(
+        request, 'user_data', None)
+    with dynamic_client(user_data=user_data) as instance:
+        yield instance
+
+
+@pytest.yield_fixture
+def client(request, fixture_utils):
+    """Provide a client that runs for every test."""
+    with _client(request, fixture_utils) as client:
+        yield client
+
+
+@pytest.yield_fixture(scope='module')
+def module_client(request, fixture_utils):
+    """Provide a client that runs once per module."""
+    with _client(request, fixture_utils) as client:
+        yield client
+
+
+@pytest.yield_fixture(scope='class')
+def class_client(request, fixture_utils):
+    """Provide a client that runs once per class."""
+    with _client(request, fixture_utils) as client:
+        yield client
diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py
new file mode 100644
index 00000000..ddd587db
--- /dev/null
+++ b/tests/integration_tests/integration_settings.py
@@ -0,0 +1,95 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+import os
+
+##################################################################
+# LAUNCH SETTINGS
+##################################################################
+
+# Keep instance (mostly for debugging) when test is finished
+KEEP_INSTANCE = False
+
+# One of:
+#  lxd_container
+#  ec2
+#  gce
+#  oci
+PLATFORM = 'lxd_container'
+
+# The cloud-specific instance type to run. E.g., a1.medium on AWS
+# If the pycloudlib instance provides a default, this can be left None
+INSTANCE_TYPE = None
+
+# Determines the base image to use or generate new images from.
+# Can be the name of the OS if running a stock image,
+# otherwise the id of the image being used if using a custom image
+OS_IMAGE = 'focal'
+
+# Populate if you want to use a pre-launched instance instead of
+# creating a new one. The exact contents will be platform dependent
+EXISTING_INSTANCE_ID = None
+
+##################################################################
+# IMAGE GENERATION SETTINGS
+##################################################################
+
+# Depending on where we are in the development / test / SRU cycle, we'll want
+# different methods of getting the source code to our SUT. Because of
+# this there are a number of different ways to initialize
+# the target environment.
+
+# Can be any of the following:
+# NONE
+#   Don't modify the target environment at all. This will run
+#   cloud-init with whatever code was baked into the image
+# IN_PLACE
+#   LXD CONTAINER only. Mount the source code as-is directly into
+#   the container to override the pre-existing cloudinit module. This
+#   won't work for non-local LXD remotes and won't run any installation
+#   code.
+# PROPOSED
+#   Install from the Ubuntu proposed repo
+# <ppa repo>, e.g., ppa:cloud-init-dev/proposed
+#   Install from a PPA. It MUST start with 'ppa:'
+# <file path>
+#   A path to a valid package to be uploaded and installed
+CLOUD_INIT_SOURCE = 'NONE'
+
+##################################################################
+# GCE SPECIFIC SETTINGS
+##################################################################
+# Required for GCE
+GCE_PROJECT = None
+
+# You probably want to override these
+GCE_REGION = 'us-central1'
+GCE_ZONE = 'a'
+
+##################################################################
+# OCI SPECIFIC SETTINGS
+##################################################################
+# Compartment-id found at
+# https://console.us-phoenix-1.oraclecloud.com/a/identity/compartments
+# Required for Oracle
+OCI_COMPARTMENT_ID = None
+
+##################################################################
+# USER SETTINGS OVERRIDES
+##################################################################
+# Bring in any user-file defined settings
+try:
+    from tests.integration_tests.user_settings import *  # noqa
+except ImportError:
+    pass
+
+##################################################################
+# ENVIRONMENT SETTINGS OVERRIDES
+##################################################################
+# Any of the settings in this file can be overridden with an
+# environment variable of the same name prepended with CLOUD_INIT_
+# E.g., CLOUD_INIT_PLATFORM
+# Perhaps a bit too hacky, but it works :)
+current_settings = [var for var in locals() if var.isupper()]
+for setting in current_settings:
+    globals()[setting] = os.getenv(
+        'CLOUD_INIT_{}'.format(setting), globals()[setting]
+    )
diff --git a/tests/integration_tests/platforms.py b/tests/integration_tests/platforms.py
new file mode 100644
index 00000000..b42414b9
--- /dev/null
+++ b/tests/integration_tests/platforms.py
@@ -0,0 +1,235 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+from abc import ABC, abstractmethod
+import logging
+import os
+from tempfile import NamedTemporaryFile
+
+from pycloudlib import EC2, GCE, Azure, OCI, LXD
+from pycloudlib.cloud import BaseCloud
+from pycloudlib.instance import BaseInstance
+
+import cloudinit
+from cloudinit.subp import subp
+from tests.integration_tests import integration_settings
+
+try:
+    from typing import Callable, Optional
+except ImportError:
+    pass
+
+
+log = logging.getLogger('integration_testing')
+
+
+class IntegrationClient(ABC):
+    client = None  # type: Optional[BaseCloud]
+    instance = None  # type: Optional[BaseInstance]
+    datasource = None  # type: Optional[str]
+    use_sudo = True
+    current_image = None
+
+    def __init__(self, user_data=None, instance_type=None, wait=True,
+                 settings=integration_settings, launch_kwargs=None):
+        self.user_data = user_data
+        self.instance_type = settings.INSTANCE_TYPE if \
+            instance_type is None else instance_type
+        self.wait = wait
+        self.settings = settings
+        self.launch_kwargs = launch_kwargs if launch_kwargs else {}
+        self.client = self._get_client()
+
+    @abstractmethod
+    def _get_client(self):
+        raise NotImplementedError
+
+    def _get_image(self):
+        if self.current_image:
+            return self.current_image
+        image_id = self.settings.OS_IMAGE
+        try:
+            image_id = self.client.released_image(self.settings.OS_IMAGE)
+        except (ValueError, IndexError):
+            pass
+        return image_id
+
+    def launch(self):
+        if self.settings.EXISTING_INSTANCE_ID:
+            log.info(
+                'Not launching instance due to EXISTING_INSTANCE_ID. '
+                'Instance id: %s', self.settings.EXISTING_INSTANCE_ID)
+            self.instance = self.client.get_instance(
+                self.settings.EXISTING_INSTANCE_ID
+            )
+            return
+        image_id = self._get_image()
+        launch_args = {
+            'image_id': image_id,
+            'user_data': self.user_data,
+            'wait': self.wait,
+        }
+        if self.instance_type:
+            launch_args['instance_type'] = self.instance_type
+        launch_args.update(self.launch_kwargs)
+        self.instance = self.client.launch(**launch_args)
+        log.info('Launched instance: %s', self.instance)
+
+    def destroy(self):
+        self.instance.delete()
+
+    def execute(self, command):
+        return self.instance.execute(command)
+
+    def pull_file(self, remote_file, local_file):
+        self.instance.pull_file(remote_file, local_file)
+
+    def push_file(self, local_path, remote_path):
+        self.instance.push_file(local_path, remote_path)
+
+    def read_from_file(self, remote_path) -> str:
+        tmp_file = NamedTemporaryFile('r')
+        self.pull_file(remote_path, tmp_file.name)
+        with tmp_file as f:
+            contents = f.read()
+        return contents
+
+    def write_to_file(self, remote_path, contents: str):
+        # Writes file locally and then pushes it rather
+        # than writing the file directly on the instance
+        with NamedTemporaryFile('w', delete=False) as tmp_file:
+            tmp_file.write(contents)
+
+        try:
+            self.push_file(tmp_file.name, remote_path)
+        finally:
+            os.unlink(tmp_file.name)
+
+    def snapshot(self):
+        return self.client.snapshot(self.instance, clean=True)
+
+    def _install_new_cloud_init(self, remote_script):
+        self.execute(remote_script)
+        version = self.execute('cloud-init -v').split()[-1]
+        log.info('Installed cloud-init version: %s', version)
+        self.instance.clean()
+        image_id = self.snapshot()
+        log.info('Created new image: %s', image_id)
+        IntegrationClient.current_image = image_id
+
+    def install_proposed_image(self):
+        log.info('Installing proposed image')
+        remote_script = (
+            '{sudo} echo deb "http://archive.ubuntu.com/ubuntu '
+            '$(lsb_release -sc)-proposed main" | '
+            '{sudo} tee /etc/apt/sources.list.d/proposed.list\n'
+            '{sudo} apt-get update -q\n'
+            '{sudo} apt-get install -qy cloud-init'
+        ).format(sudo='sudo' if self.use_sudo else '')
+        self._install_new_cloud_init(remote_script)
+
+    def install_ppa(self, repo):
+        log.info('Installing PPA')
+        remote_script = (
+            '{sudo} add-apt-repository {repo} -y && '
+            '{sudo} apt-get update -q && '
+            '{sudo} apt-get install -qy cloud-init'
+        ).format(sudo='sudo' if self.use_sudo else '', repo=repo)
+        self._install_new_cloud_init(remote_script)
+
+    def install_deb(self):
+        log.info('Installing deb package')
+        deb_path = integration_settings.CLOUD_INIT_SOURCE
+        deb_name = os.path.basename(deb_path)
+        remote_path = '/var/tmp/{}'.format(deb_name)
+        self.push_file(
+            local_path=integration_settings.CLOUD_INIT_SOURCE,
+            remote_path=remote_path)
+        remote_script = '{sudo} dpkg -i {path}'.format(
+            sudo='sudo' if self.use_sudo else '', path=remote_path)
+        self._install_new_cloud_init(remote_script)
+
+    def __enter__(self):
+        self.launch()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if not self.settings.KEEP_INSTANCE:
+            self.destroy()
+
+
+class Ec2Client(IntegrationClient):
+    datasource = 'ec2'
+
+    def _get_client(self):
+        return EC2(tag='ec2-integration-test')
+
+
+class GceClient(IntegrationClient):
+    datasource = 'gce'
+
+    def _get_client(self):
+        return GCE(
+            tag='gce-integration-test',
+            project=self.settings.GCE_PROJECT,
+            region=self.settings.GCE_REGION,
+            zone=self.settings.GCE_ZONE,
+        )
+
+
+class AzureClient(IntegrationClient):
+    datasource = 'azure'
+
+    def _get_client(self):
+        return Azure(tag='azure-integration-test')
+
+
+class OciClient(IntegrationClient):
+    datasource = 'oci'
+
+    def _get_client(self):
+        return OCI(
+            tag='oci-integration-test',
+            compartment_id=self.settings.OCI_COMPARTMENT_ID
+        )
+
+
+class LxdContainerClient(IntegrationClient):
+    datasource = 'lxd_container'
+    use_sudo = False
+
+    def _get_client(self):
+        return LXD(tag='lxd-integration-test')
+
+    def _mount_source(self):
+        command = (
+            'lxc config device add {name} host-cloud-init disk '
+            'source={cloudinit_path} '
+            'path=/usr/lib/python3/dist-packages/cloudinit'
+        ).format(
+            name=self.instance.name, cloudinit_path=cloudinit.__path__[0])
+        subp(command.split())
+
+    def launch(self):
+        super().launch()
+        if self.settings.CLOUD_INIT_SOURCE == 'IN_PLACE':
+            self._mount_source()
+
+
+client_name_to_class = {
+    'ec2': Ec2Client,
+    'gce': GceClient,
+    # 'azure': AzureClient,  # Not supported yet
+    'oci': OciClient,
+    'lxd_container': LxdContainerClient
+}
+
+try:
+    dynamic_client = client_name_to_class[
+        integration_settings.PLATFORM
+    ]  # type: Callable[..., IntegrationClient]
+except KeyError:
+    raise ValueError(
+        "{} is an invalid PLATFORM specified in settings. "
+        "Must be one of {}".format(
+            integration_settings.PLATFORM, list(client_name_to_class.keys())
+        )
+    )
diff --git a/tox.ini b/tox.ini
index a92c63e0..3bc83a2a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -139,8 +139,15 @@ deps =
 
 [pytest]
 # TODO: s/--strict/--strict-markers/ once xenial support is dropped
+testpaths = cloudinit tests/unittests
 addopts = --strict
 markers =
     allow_subp_for: allow subp usage for the given commands (disable_subp_usage)
     allow_all_subp: allow all subp usage (disable_subp_usage)
     ds_sys_cfg: a sys_cfg dict to be used by datasource fixtures
+    ec2: test will only run on EC2 platform
+    gce: test will only run on GCE platform
+    azure: test will only run on Azure platform
+    oci: test will only run on OCI platform
+    lxd_container: test will only run in LXD container
+    user_data: the user data to be passed to the test instance