diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | HACKING.rst | 11 | ||||
| -rw-r--r-- | doc/rtd/index.rst | 3 | ||||
| -rw-r--r-- | doc/rtd/topics/cloud_tests.rst (renamed from doc/rtd/topics/tests.rst) | 9 | ||||
| -rw-r--r-- | doc/rtd/topics/integration_tests.rst | 81 | ||||
| -rw-r--r-- | integration-requirements.txt | 2 | ||||
| -rw-r--r-- | tests/integration_tests/conftest.py | 106 | ||||
| -rw-r--r-- | tests/integration_tests/integration_settings.py | 95 | ||||
| -rw-r--r-- | tests/integration_tests/platforms.py | 235 | ||||
| -rw-r--r-- | tox.ini | 7 |
10 files changed, 547 insertions, 5 deletions
@@ -27,3 +27,6 @@ cloud-init_*.dsc cloud-init_*.orig.tar.gz cloud-init_*.tar.xz cloud-init_*.upload + +# user test settings +tests/integration_tests/user_settings.py diff --git a/HACKING.rst b/HACKING.rst index 60c7b5e0..4ae7f7b4 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -173,9 +173,18 @@ Cloud Config Modules * Any new modules should use underscores in any new config options and not hyphens (e.g. `new_option` and *not* `new-option`). -Unit Testing +.. _unit_testing: + +Testing ------------ +cloud-init has both unit tests and integration tests. Unit tests can +be found in-tree alongside the source code, as well as +at ``tests/unittests``. Integration tests can be found at +``tests/integration_tests``. Documentation specifically for integration +tests can be found on the :ref:`integration_tests` page, but +the guidelines specified below apply to both types of tests. + cloud-init uses `pytest`_ to run its tests, and has tests written both as ``unittest.TestCase`` sub-classes and as un-subclassed pytest tests. The following guidelines should be followed: diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst index 0015e35a..ddcb0b31 100644 --- a/doc/rtd/index.rst +++ b/doc/rtd/index.rst @@ -75,6 +75,7 @@ Having trouble? We would like to help! topics/dir_layout.rst topics/analyze.rst topics/docs.rst - topics/tests.rst + topics/integration_tests.rst + topics/cloud_tests.rst .. vi: textwidth=79 diff --git a/doc/rtd/topics/tests.rst b/doc/rtd/topics/cloud_tests.rst index f03b5969..e4e893d2 100644 --- a/doc/rtd/topics/tests.rst +++ b/doc/rtd/topics/cloud_tests.rst @@ -1,6 +1,9 @@ -******************* -Integration Testing -******************* +************************ +Cloud tests (Deprecated) +************************ + +Cloud tests are longer be maintained. For writing integration +tests, see the :ref:`integration_tests` page. Overview ======== diff --git a/doc/rtd/topics/integration_tests.rst b/doc/rtd/topics/integration_tests.rst new file mode 100644 index 00000000..aeda326c --- /dev/null +++ b/doc/rtd/topics/integration_tests.rst @@ -0,0 +1,81 @@ +.. _integration_tests: + +******************* +Integration Testing +******************* + +Overview +========= + +Integration tests are written using pytest and are located at +``tests/integration_tests``. General design principles +laid out in :ref:`unit_testing` should be followed for integration tests. + +Setup is accomplished via a set of fixtures located in +``tests/integration_tests/conftest.py``. + +Image Setup +=========== + +Image setup occurs once when a test session begins and is implemented +via fixture. Image setup roughly follows these steps: + +* Launch an instance on the specified test platform +* Install the version of cloud-init under test +* Run ``cloud-init clean`` on the instance so subsequent boots + resemble out of the box behavior +* Take a snapshot of the instance to be used as a new image from + which new instances can be launched + +Test Setup +============== +Test setup occurs between image setup and test execution. Test setup +is implemented via one of the ``client`` fixtures. When a client fixture +is used, a test instance from which to run tests is launched prior to +test execution and torn down after. + +Test Definition +=============== +Tests are defined like any other pytest test. The ``user_data`` +mark can be used to supply the cloud-config user data. Platform specific +marks can be used to limit tests to particular platforms. The +client fixture can be used to interact with the launched +test instance. + +A basic example: + +.. code-block:: python + + USER_DATA = """#cloud-config + bootcmd: + - echo 'hello config!' > /tmp/user_data.txt""" + + + class TestSimple: + @pytest.mark.user_data(USER_DATA) + @pytest.mark.ec2 + def test_simple(self, client): + print(client.exec('cloud-init -v')) + +Test Execution +============== +Test execution happens via pytest. To run all integration tests, +you would run: + +.. code-block:: bash + + pytest tests/integration_tests/ + + +Configuration +============= + +All possible configuration values are defined in +``tests/integration_tests/integration_settings.py``. Defaults can be +overridden by supplying values in ``tests/integration_tests/user_settings.py`` +or by providing an environment variable of the same name prepended with +``CLOUD_INIT_``. For example, to set the ``PLATFORM`` setting: + +.. code-block:: bash + + CLOUD_INIT_PLATFORM='ec2' pytest tests/integration_tests/ diff --git a/integration-requirements.txt b/integration-requirements.txt index 13cfb9d7..64455c79 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -4,6 +4,8 @@ # Note: Changes to this requirements may require updates to # the packages/pkg-deps.json file as well. # +pytest +git+https://github.com/canonical/pycloudlib.git # ec2 backend boto3==1.14.53 diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py new file mode 100644 index 00000000..a170bfc9 --- /dev/null +++ b/tests/integration_tests/conftest.py @@ -0,0 +1,106 @@ +# This file is part of cloud-init. See LICENSE file for license information. +import os +import logging +import pytest +import sys +from contextlib import contextmanager + +from tests.integration_tests import integration_settings +from tests.integration_tests.platforms import ( + dynamic_client, + LxdContainerClient, + client_name_to_class +) + +log = logging.getLogger('integration_testing') +log.addHandler(logging.StreamHandler(sys.stdout)) +log.setLevel(logging.INFO) + + +def pytest_runtest_setup(item): + """Skip tests on unsupported clouds. + + A test can take any number of marks to specify the platforms it can + run on. If a platform(s) is specified and we're not running on that + platform, then skip the test. If platform specific marks are not + specified, then we assume the test can be run anywhere. + """ + all_platforms = client_name_to_class.keys() + supported_platforms = set(all_platforms).intersection( + mark.name for mark in item.iter_markers()) + current_platform = integration_settings.PLATFORM + if supported_platforms and current_platform not in supported_platforms: + pytest.skip('Cannot run on platform {}'.format(current_platform)) + + +# disable_subp_usage is defined at a higher level, but we don't +# want it applied here +@pytest.fixture() +def disable_subp_usage(request): + pass + + +@pytest.fixture(scope='session', autouse=True) +def setup_image(): + """Setup the target environment with the correct version of cloud-init. + + So we can launch instances / run tests with the correct image + """ + client = dynamic_client() + log.info('Setting up environment for %s', client.datasource) + if integration_settings.CLOUD_INIT_SOURCE == 'NONE': + pass # that was easy + elif integration_settings.CLOUD_INIT_SOURCE == 'IN_PLACE': + if not isinstance(client, LxdContainerClient): + raise ValueError( + 'IN_PLACE as CLOUD_INIT_SOURCE only works for LXD') + # The mount needs to happen after the instance is launched, so + # no further action needed here + elif integration_settings.CLOUD_INIT_SOURCE == 'PROPOSED': + client.launch() + client.install_proposed_image() + elif integration_settings.CLOUD_INIT_SOURCE.startswith('ppa:'): + client.launch() + client.install_ppa(integration_settings.CLOUD_INIT_SOURCE) + elif os.path.isfile(str(integration_settings.CLOUD_INIT_SOURCE)): + client.launch() + client.install_deb() + if client.instance: + # Even if we're keeping instances, we don't want to keep this + # one around as it was just for image creation + client.destroy() + log.info('Done with environment setup') + + +@contextmanager +def _client(request, fixture_utils): + """Fixture implementation for the client fixtures. + + Launch the dynamic IntegrationClient instance using any provided + userdata, yield to the test, then cleanup + """ + user_data = fixture_utils.closest_marker_first_arg_or( + request, 'user_data', None) + with dynamic_client(user_data=user_data) as instance: + yield instance + + +@pytest.yield_fixture +def client(request, fixture_utils): + """Provide a client that runs for every test.""" + with _client(request, fixture_utils) as client: + yield client + + +@pytest.yield_fixture(scope='module') +def module_client(request, fixture_utils): + """Provide a client that runs once per module.""" + with _client(request, fixture_utils) as client: + yield client + + +@pytest.yield_fixture(scope='class') +def class_client(request, fixture_utils): + """Provide a client that runs once per class.""" + with _client(request, fixture_utils) as client: + yield client diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py new file mode 100644 index 00000000..ddd587db --- /dev/null +++ b/tests/integration_tests/integration_settings.py @@ -0,0 +1,95 @@ +# This file is part of cloud-init. See LICENSE file for license information. +import os + +################################################################## +# LAUNCH SETTINGS +################################################################## + +# Keep instance (mostly for debugging) when test is finished +KEEP_INSTANCE = False + +# One of: +# lxd_container +# ec2 +# gce +# oci +PLATFORM = 'lxd_container' + +# The cloud-specific instance type to run. E.g., a1.medium on AWS +# If the pycloudlib instance provides a default, this can be left None +INSTANCE_TYPE = None + +# Determines the base image to use or generate new images from. +# Can be the name of the OS if running a stock image, +# otherwise the id of the image being used if using a custom image +OS_IMAGE = 'focal' + +# Populate if you want to use a pre-launched instance instead of +# creating a new one. The exact contents will be platform dependent +EXISTING_INSTANCE_ID = None + +################################################################## +# IMAGE GENERATION SETTINGS +################################################################## + +# Depending on where we are in the development / test / SRU cycle, we'll want +# different methods of getting the source code to our SUT. Because of +# this there are a number of different ways to initialize +# the target environment. + +# Can be any of the following: +# NONE +# Don't modify the target environment at all. This will run +# cloud-init with whatever code was baked into the image +# IN_PLACE +# LXD CONTAINER only. Mount the source code as-is directly into +# the container to override the pre-existing cloudinit module. This +# won't work for non-local LXD remotes and won't run any installation +# code. +# PROPOSED +# Install from the Ubuntu proposed repo +# <ppa repo>, e.g., ppa:cloud-init-dev/proposed +# Install from a PPA. It MUST start with 'ppa:' +# <file path> +# A path to a valid package to be uploaded and installed +CLOUD_INIT_SOURCE = 'NONE' + +################################################################## +# GCE SPECIFIC SETTINGS +################################################################## +# Required for GCE +GCE_PROJECT = None + +# You probably want to override these +GCE_REGION = 'us-central1' +GCE_ZONE = 'a' + +################################################################## +# OCI SPECIFIC SETTINGS +################################################################## +# Compartment-id found at +# https://console.us-phoenix-1.oraclecloud.com/a/identity/compartments +# Required for Oracle +OCI_COMPARTMENT_ID = None + +################################################################## +# USER SETTINGS OVERRIDES +################################################################## +# Bring in any user-file defined settings +try: + from tests.integration_tests.user_settings import * # noqa +except ImportError: + pass + +################################################################## +# ENVIRONMENT SETTINGS OVERRIDES +################################################################## +# Any of the settings in this file can be overridden with an +# environment variable of the same name prepended with CLOUD_INIT_ +# E.g., CLOUD_INIT_PLATFORM +# Perhaps a bit too hacky, but it works :) +current_settings = [var for var in locals() if var.isupper()] +for setting in current_settings: + globals()[setting] = os.getenv( + 'CLOUD_INIT_{}'.format(setting), globals()[setting] + ) diff --git a/tests/integration_tests/platforms.py b/tests/integration_tests/platforms.py new file mode 100644 index 00000000..b42414b9 --- /dev/null +++ b/tests/integration_tests/platforms.py @@ -0,0 +1,235 @@ +# This file is part of cloud-init. See LICENSE file for license information. +from abc import ABC, abstractmethod +import logging +import os +from tempfile import NamedTemporaryFile + +from pycloudlib import EC2, GCE, Azure, OCI, LXD +from pycloudlib.cloud import BaseCloud +from pycloudlib.instance import BaseInstance + +import cloudinit +from cloudinit.subp import subp +from tests.integration_tests import integration_settings + +try: + from typing import Callable, Optional +except ImportError: + pass + + +log = logging.getLogger('integration_testing') + + +class IntegrationClient(ABC): + client = None # type: Optional[BaseCloud] + instance = None # type: Optional[BaseInstance] + datasource = None # type: Optional[str] + use_sudo = True + current_image = None + + def __init__(self, user_data=None, instance_type=None, wait=True, + settings=integration_settings, launch_kwargs=None): + self.user_data = user_data + self.instance_type = settings.INSTANCE_TYPE if \ + instance_type is None else instance_type + self.wait = wait + self.settings = settings + self.launch_kwargs = launch_kwargs if launch_kwargs else {} + self.client = self._get_client() + + @abstractmethod + def _get_client(self): + raise NotImplementedError + + def _get_image(self): + if self.current_image: + return self.current_image + image_id = self.settings.OS_IMAGE + try: + image_id = self.client.released_image(self.settings.OS_IMAGE) + except (ValueError, IndexError): + pass + return image_id + + def launch(self): + if self.settings.EXISTING_INSTANCE_ID: + log.info( + 'Not launching instance due to EXISTING_INSTANCE_ID. ' + 'Instance id: %s', self.settings.EXISTING_INSTANCE_ID) + self.instance = self.client.get_instance( + self.settings.EXISTING_INSTANCE_ID + ) + return + image_id = self._get_image() + launch_args = { + 'image_id': image_id, + 'user_data': self.user_data, + 'wait': self.wait, + } + if self.instance_type: + launch_args['instance_type'] = self.instance_type + launch_args.update(self.launch_kwargs) + self.instance = self.client.launch(**launch_args) + log.info('Launched instance: %s', self.instance) + + def destroy(self): + self.instance.delete() + + def execute(self, command): + return self.instance.execute(command) + + def pull_file(self, remote_file, local_file): + self.instance.pull_file(remote_file, local_file) + + def push_file(self, local_path, remote_path): + self.instance.push_file(local_path, remote_path) + + def read_from_file(self, remote_path) -> str: + tmp_file = NamedTemporaryFile('r') + self.pull_file(remote_path, tmp_file.name) + with tmp_file as f: + contents = f.read() + return contents + + def write_to_file(self, remote_path, contents: str): + # Writes file locally and then pushes it rather + # than writing the file directly on the instance + with NamedTemporaryFile('w', delete=False) as tmp_file: + tmp_file.write(contents) + + try: + self.push_file(tmp_file.name, remote_path) + finally: + os.unlink(tmp_file.name) + + def snapshot(self): + return self.client.snapshot(self.instance, clean=True) + + def _install_new_cloud_init(self, remote_script): + self.execute(remote_script) + version = self.execute('cloud-init -v').split()[-1] + log.info('Installed cloud-init version: %s', version) + self.instance.clean() + image_id = self.snapshot() + log.info('Created new image: %s', image_id) + IntegrationClient.current_image = image_id + + def install_proposed_image(self): + log.info('Installing proposed image') + remote_script = ( + '{sudo} echo deb "http://archive.ubuntu.com/ubuntu ' + '$(lsb_release -sc)-proposed main" | ' + '{sudo} tee /etc/apt/sources.list.d/proposed.list\n' + '{sudo} apt-get update -q\n' + '{sudo} apt-get install -qy cloud-init' + ).format(sudo='sudo' if self.use_sudo else '') + self._install_new_cloud_init(remote_script) + + def install_ppa(self, repo): + log.info('Installing PPA') + remote_script = ( + '{sudo} add-apt-repository {repo} -y && ' + '{sudo} apt-get update -q && ' + '{sudo} apt-get install -qy cloud-init' + ).format(sudo='sudo' if self.use_sudo else '', repo=repo) + self._install_new_cloud_init(remote_script) + + def install_deb(self): + log.info('Installing deb package') + deb_path = integration_settings.CLOUD_INIT_SOURCE + deb_name = os.path.basename(deb_path) + remote_path = '/var/tmp/{}'.format(deb_name) + self.push_file( + local_path=integration_settings.CLOUD_INIT_SOURCE, + remote_path=remote_path) + remote_script = '{sudo} dpkg -i {path}'.format( + sudo='sudo' if self.use_sudo else '', path=remote_path) + self._install_new_cloud_init(remote_script) + + def __enter__(self): + self.launch() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.settings.KEEP_INSTANCE: + self.destroy() + + +class Ec2Client(IntegrationClient): + datasource = 'ec2' + + def _get_client(self): + return EC2(tag='ec2-integration-test') + + +class GceClient(IntegrationClient): + datasource = 'gce' + + def _get_client(self): + return GCE( + tag='gce-integration-test', + project=self.settings.GCE_PROJECT, + region=self.settings.GCE_REGION, + zone=self.settings.GCE_ZONE, + ) + + +class AzureClient(IntegrationClient): + datasource = 'azure' + + def _get_client(self): + return Azure(tag='azure-integration-test') + + +class OciClient(IntegrationClient): + datasource = 'oci' + + def _get_client(self): + return OCI( + tag='oci-integration-test', + compartment_id=self.settings.OCI_COMPARTMENT_ID + ) + + +class LxdContainerClient(IntegrationClient): + datasource = 'lxd_container' + use_sudo = False + + def _get_client(self): + return LXD(tag='lxd-integration-test') + + def _mount_source(self): + command = ( + 'lxc config device add {name} host-cloud-init disk ' + 'source={cloudinit_path} ' + 'path=/usr/lib/python3/dist-packages/cloudinit' + ).format( + name=self.instance.name, cloudinit_path=cloudinit.__path__[0]) + subp(command.split()) + + def launch(self): + super().launch() + if self.settings.CLOUD_INIT_SOURCE == 'IN_PLACE': + self._mount_source() + + +client_name_to_class = { + 'ec2': Ec2Client, + 'gce': GceClient, + # 'azure': AzureClient, # Not supported yet + 'oci': OciClient, + 'lxd_container': LxdContainerClient +} + +try: + dynamic_client = client_name_to_class[ + integration_settings.PLATFORM + ] # type: Callable[..., IntegrationClient] +except KeyError: + raise ValueError( + "{} is an invalid PLATFORM specified in settings. " + "Must be one of {}".format( + integration_settings.PLATFORM, list(client_name_to_class.keys()) + ) + ) @@ -139,8 +139,15 @@ deps = [pytest] # TODO: s/--strict/--strict-markers/ once xenial support is dropped +testpaths = cloudinit tests/unittests addopts = --strict markers = allow_subp_for: allow subp usage for the given commands (disable_subp_usage) allow_all_subp: allow all subp usage (disable_subp_usage) ds_sys_cfg: a sys_cfg dict to be used by datasource fixtures + ec2: test will only run on EC2 platform + gce: test will only run on GCE platform + azure: test will only run on Azure platform + oci: test will only run on OCI platform + lxd_container: test will only run in LXD container + user_data: the user data to be passed to the test instance |