integration_tests: introduce skipping of tests by OS (#702)

This introduces an optional, more complex OS_IMAGE format (`<image
id>::<os>::<release>`) which allows the specification of the OS/OS
release which the given image ID corresponds to.  This information is
used to skip tests which do not apply to the image.
 
This commit is comprised of the following discrete changes:
* introduce the IntegrationImage class, to handle parsing and storing
   the new OS_IMAGE format
* support inferring the OS and OS release of Ubuntu series, so that we
  can continue to set OS_IMAGE to just a series name and have test
  skipping work
* add documentation on Image Selection to integration_tests.rst
* introduce the actual skipping behaviour based on OS marks
* apply the `ubuntu` mark to all tests that should be skipped on
  non-Ubuntu operating systems

10 files changed, 122 insertions, 10 deletions

diff --git a/doc/rtd/topics/integration_tests.rst b/doc/rtd/topics/integration_tests.rst
index aeda326c..3cfca31e 100644
--- a/doc/rtd/topics/integration_tests.rst
+++ b/doc/rtd/topics/integration_tests.rst
@@ -14,6 +14,36 @@ laid out in :ref:`unit_testing` should be followed for integration tests.
 Setup is accomplished via a set of fixtures located in
 ``tests/integration_tests/conftest.py``.
 
+Image Selection
+===============
+
+Each integration testing run uses a single image as its basis.  This
+image is configured using the ``OS_IMAGE`` variable; see
+:ref:`Configuration` for details of how configuration works.
+
+``OS_IMAGE`` can take two types of value: an Ubuntu series name (e.g.
+"focal"), or an image specification.  If an Ubuntu series name is
+given, then the most recent image for that series on the target cloud
+will be used.  For other use cases, an image specification is used.
+
+In its simplest form, an image specification can simply be a cloud's
+image ID (e.g. "ami-deadbeef", "ubuntu:focal").  In this case, the
+image so-identified will be used as the basis for this testing run.
+
+This has a drawback, however: as we do not know what OS or release is
+within the image, the integration testing framework will run *all*
+tests against the image in question.  If it's a RHEL8 image, then we
+would expect Ubuntu-specific tests to fail (and vice versa).
+
+To address this, a full image specification can be given.  This is of
+the form: ``<image_id>[::<os>[::<release]]`` where ``image_id`` is a
+cloud's image ID, ``os`` is the OS name, and ``release`` is the OS
+release name.  So, for example, Ubuntu 18.04 (Bionic Beaver) on LXD is
+``ubuntu:bionic::ubuntu::bionic`` or RHEL 8 on Amazon is
+``ami-justanexample::rhel::8``.  When a full specification is given,
+only tests which are intended for use on that OS and release will be
+executed.
+
 Image Setup
 ===========
 
diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py
index 4d5c2c2a..8cdb3fcc 100644
--- a/tests/integration_tests/clouds.py
+++ b/tests/integration_tests/clouds.py
@@ -6,7 +6,7 @@ from pycloudlib import EC2, GCE, Azure, OCI, LXDContainer, LXDVirtualMachine
 from pycloudlib.lxd.instance import LXDInstance
 
 import cloudinit
-from cloudinit.subp import subp
+from cloudinit.subp import subp, ProcessExecutionError
 from tests.integration_tests import integration_settings
 from tests.integration_tests.instances import (
     IntegrationEc2Instance,
@@ -25,6 +25,65 @@ except ImportError:
 log = logging.getLogger('integration_testing')
 
 
+def _get_ubuntu_series() -> list:
+    """Use distro-info-data's ubuntu.csv to get a list of Ubuntu series"""
+    out = ""
+    try:
+        out, _err = subp(["ubuntu-distro-info", "-a"])
+    except ProcessExecutionError:
+        log.info(
+            "ubuntu-distro-info (from the distro-info package) must be"
+            " installed to guess Ubuntu os/release"
+        )
+    return out.splitlines()
+
+
+class ImageSpecification:
+    """A specification of an image to launch for testing.
+
+    If either of ``os`` and ``release`` are not specified, an attempt will be
+    made to infer the correct values for these on instantiation.
+
+    :param image_id:
+        The image identifier used by the rest of the codebase to launch this
+        image.
+    :param os:
+        An optional string describing the operating system this image is for
+        (e.g.  "ubuntu", "rhel", "freebsd").
+    :param release:
+        A optional string describing the operating system release (e.g.
+        "focal", "8"; the exact values here will depend on the OS).
+    """
+
+    def __init__(
+        self,
+        image_id: str,
+        os: "Optional[str]" = None,
+        release: "Optional[str]" = None,
+    ):
+        if image_id in _get_ubuntu_series():
+            if os is None:
+                os = "ubuntu"
+            if release is None:
+                release = image_id
+
+        self.image_id = image_id
+        self.os = os
+        self.release = release
+        log.info(
+            "Detected image: image_id=%s os=%s release=%s",
+            self.image_id,
+            self.os,
+            self.release,
+        )
+
+    @classmethod
+    def from_os_image(cls):
+        """Return an ImageSpecification for integration_settings.OS_IMAGE."""
+        parts = integration_settings.OS_IMAGE.split("::", 2)
+        return cls(*parts)
+
+
 class IntegrationCloud(ABC):
     datasource = None  # type: Optional[str]
     integration_instance_cls = IntegrationInstance
@@ -57,13 +116,11 @@ class IntegrationCloud(ABC):
         raise NotImplementedError
 
     def _get_initial_image(self):
-        _released_image_id = self.settings.OS_IMAGE
+        image = ImageSpecification.from_os_image()
         try:
-            _released_image_id = self.cloud_instance.released_image(
-                self.settings.OS_IMAGE)
+            return self.cloud_instance.released_image(image.image_id)
         except (ValueError, IndexError):
-            pass
-        return _released_image_id
+            return image.image_id
 
     def _perform_launch(self, launch_kwargs):
         pycloudlib_instance = self.cloud_instance.launch(**launch_kwargs)
diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py
index d7e0fca2..cc545b0f 100644
--- a/tests/integration_tests/conftest.py
+++ b/tests/integration_tests/conftest.py
@@ -10,12 +10,13 @@ from pathlib import Path
 
 from tests.integration_tests import integration_settings
 from tests.integration_tests.clouds import (
+    AzureCloud,
     Ec2Cloud,
     GceCloud,
-    AzureCloud,
-    OciCloud,
+    ImageSpecification,
     LxdContainerCloud,
     LxdVmCloud,
+    OciCloud,
 )
 from tests.integration_tests.instances import IntegrationInstance
 
@@ -32,6 +33,7 @@ platforms = {
     'lxd_container': LxdContainerCloud,
     'lxd_vm': LxdVmCloud,
 }
+os_list = ["ubuntu"]
 
 session_start_time = datetime.datetime.now().strftime('%y%m%d%H%M%S')
 
@@ -60,6 +62,12 @@ def pytest_runtest_setup(item):
     if supported_platforms and current_platform not in supported_platforms:
         pytest.skip(unsupported_message)
 
+    image = ImageSpecification.from_os_image()
+    current_os = image.os
+    supported_os_set = set(os_list).intersection(test_marks)
+    if current_os and supported_os_set and current_os not in supported_os_set:
+        pytest.skip("Cannot run on OS {}".format(current_os))
+
 
 # disable_subp_usage is defined at a higher level, but we don't
 # want it applied here
diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py
index 94d54f74..07a6d541 100644
--- a/tests/integration_tests/integration_settings.py
+++ b/tests/integration_tests/integration_settings.py
@@ -22,8 +22,11 @@ PLATFORM = 'lxd_container'
 INSTANCE_TYPE = None
 
 # Determines the base image to use or generate new images from.
-# Can be the name of the OS if running a stock image,
-# otherwise the id of the image being used if using a custom image
+#
+# This can be the name of an Ubuntu release, or in the format
+# <image_id>[::<os>[::<release>]].  If given, os and release should describe
+# the image specified by image_id.  (Ubuntu releases are converted to this
+# format internally; in this case, to "focal::ubuntu::focal".)
 OS_IMAGE = 'focal'
 
 # Populate if you want to use a pre-launched instance instead of
diff --git a/tests/integration_tests/modules/test_apt_configure_sources_list.py b/tests/integration_tests/modules/test_apt_configure_sources_list.py
index d2bcc61a..28cbe19f 100644
--- a/tests/integration_tests/modules/test_apt_configure_sources_list.py
+++ b/tests/integration_tests/modules/test_apt_configure_sources_list.py
@@ -40,6 +40,7 @@ EXPECTED_REGEXES = [
 
 
 @pytest.mark.ci
+@pytest.mark.ubuntu
 class TestAptConfigureSourcesList:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_package_update_upgrade_install.py b/tests/integration_tests/modules/test_package_update_upgrade_install.py
index 8a38ad84..28d741bc 100644
--- a/tests/integration_tests/modules/test_package_update_upgrade_install.py
+++ b/tests/integration_tests/modules/test_package_update_upgrade_install.py
@@ -26,6 +26,7 @@ package_upgrade: true
 """
 
 
+@pytest.mark.ubuntu
 @pytest.mark.user_data(USER_DATA)
 class TestPackageUpdateUpgradeInstall:
 
diff --git a/tests/integration_tests/modules/test_snap.py b/tests/integration_tests/modules/test_snap.py
index b626f6b0..481edbaa 100644
--- a/tests/integration_tests/modules/test_snap.py
+++ b/tests/integration_tests/modules/test_snap.py
@@ -20,6 +20,7 @@ snap:
 
 
 @pytest.mark.ci
+@pytest.mark.ubuntu
 class TestSnap:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_ssh_import_id.py b/tests/integration_tests/modules/test_ssh_import_id.py
index 45d37d6c..3db573b5 100644
--- a/tests/integration_tests/modules/test_ssh_import_id.py
+++ b/tests/integration_tests/modules/test_ssh_import_id.py
@@ -3,6 +3,10 @@
 This test specifies ssh keys to be imported by the ``ssh_import_id`` module
 and then checks that if the ssh keys were successfully imported.
 
+TODO:
+* This test assumes that SSH keys will be imported into the /home/ubuntu; this
+  will need modification to run on other OSes.
+
 (This is ported from
 ``tests/cloud_tests/testcases/modules/ssh_import_id.yaml``.)"""
 
@@ -18,6 +22,7 @@ ssh_import_id:
 
 
 @pytest.mark.ci
+@pytest.mark.ubuntu
 class TestSshImportId:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py
index 6a51f5a6..ee08d87b 100644
--- a/tests/integration_tests/modules/test_users_groups.py
+++ b/tests/integration_tests/modules/test_users_groups.py
@@ -2,6 +2,10 @@
 
 This test specifies a number of users and groups via user-data, and confirms
 that they have been configured correctly in the system under test.
+
+TODO:
+* This test assumes that the "ubuntu" user will be created when "default" is
+  specified; this will need modification to run on other OSes.
 """
 import re
 
@@ -41,6 +45,7 @@ AHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
 @pytest.mark.ci
 @pytest.mark.user_data(USER_DATA)
 class TestUsersGroups:
+    @pytest.mark.ubuntu
     @pytest.mark.parametrize(
         "getent_args,regex",
         [
diff --git a/tox.ini b/tox.ini
index 022b918d..df1deb6f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -179,3 +179,4 @@ markers =
     user_data: the user data to be passed to the test instance
     instance_name: the name to be used for the test instance
     sru_2020_11: test is part of the 2020/11 SRU verification
+    ubuntu: this test should run on Ubuntu