Remove 3.5 and xenial support (SC-711) (#1167)

Includes:
 - Update tox.ini and .travis.yml accordingly
 - Cleanup tox.ini with new tox syntax and cloud-init dependencies
 - Update documentation accordingly
 - Replace/remove xenial references where additional testing isn't required
 - Remove xenial checks in integration tests
 - Replace yield_fixture with fixture in pytest tests

Sections of code commented with lines like "Remove when Xenial is no
longer supported" still exist as they're require additional testing.

17 files changed, 21 insertions, 59 deletions

diff --git a/tests/integration_tests/bugs/test_gh626.py b/tests/integration_tests/bugs/test_gh626.py
index 7c720143..b80b677a 100644
--- a/tests/integration_tests/bugs/test_gh626.py
+++ b/tests/integration_tests/bugs/test_gh626.py
@@ -8,7 +8,6 @@ import pytest
 import yaml
 
 from tests.integration_tests import random_mac_address
-from tests.integration_tests.clouds import ImageSpecification
 from tests.integration_tests.instances import IntegrationInstance
 
 MAC_ADDRESS = random_mac_address()
@@ -29,7 +28,6 @@ iface eth0 inet dhcp
     ethernet-wol g"""
 
 
-@pytest.mark.sru_2020_11
 @pytest.mark.lxd_container
 @pytest.mark.lxd_vm
 @pytest.mark.lxd_config_dict(
@@ -39,11 +37,6 @@ iface eth0 inet dhcp
     }
 )
 def test_wakeonlan(client: IntegrationInstance):
-    if ImageSpecification.from_os_image().release == "xenial":
-        eni = client.execute("cat /etc/network/interfaces.d/50-cloud-init.cfg")
-        assert eni.endswith(EXPECTED_ENI_END)
-        return
-
     netplan_cfg = client.execute("cat /etc/netplan/50-cloud-init.yaml")
     netplan_yaml = yaml.safe_load(netplan_cfg)
     assert "wakeonlan" in netplan_yaml["network"]["ethernets"]["eth0"]
diff --git a/tests/integration_tests/bugs/test_lp1898997.py b/tests/integration_tests/bugs/test_lp1898997.py
index 115bd34f..d8ea54c3 100644
--- a/tests/integration_tests/bugs/test_lp1898997.py
+++ b/tests/integration_tests/bugs/test_lp1898997.py
@@ -47,8 +47,6 @@ version: 2
 @pytest.mark.lxd_vm
 @pytest.mark.lxd_use_exec
 @pytest.mark.not_bionic
-@pytest.mark.not_xenial
-@pytest.mark.sru_2020_11
 @pytest.mark.ubuntu
 class TestInterfaceListingWithOpenvSwitch:
     def test_ovs_member_interfaces_not_excluded(self, client):
diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py
index b14b6ad0..2e44ef29 100644
--- a/tests/integration_tests/conftest.py
+++ b/tests/integration_tests/conftest.py
@@ -45,17 +45,6 @@ os_list = ["ubuntu"]
 
 session_start_time = datetime.datetime.now().strftime("%y%m%d%H%M%S")
 
-XENIAL_LXD_VM_EXEC_MSG = """\
-The default xenial images do not support `exec` for LXD VMs.
-
-Specify an image known to work using:
-
-    OS_IMAGE=<image id>::ubuntu::xenial
-
-You can re-run specifically tests that require this by passing `-m
-lxd_use_exec` to pytest.
-"""
-
 
 def pytest_runtest_setup(item):
     """Skip tests on unsupported clouds.
@@ -101,7 +90,7 @@ def disable_subp_usage(request):
     pass
 
 
-@pytest.yield_fixture(scope="session")
+@pytest.fixture(scope="session")
 def session_cloud():
     if integration_settings.PLATFORM not in platforms.keys():
         raise ValueError(
@@ -246,16 +235,6 @@ def _client(request, fixture_utils, session_cloud: IntegrationCloud):
     if lxd_use_exec is not None:
         if not isinstance(session_cloud, _LxdIntegrationCloud):
             pytest.skip("lxd_use_exec requires LXD")
-        if isinstance(session_cloud, LxdVmCloud):
-            image_spec = ImageSpecification.from_os_image()
-            if image_spec.release == image_spec.image_id == "xenial":
-                # Why fail instead of skip?  We expect that skipped tests will
-                # be run in a different one of our usual battery of test runs
-                # (e.g. LXD-only tests are skipped on EC2 but will run in our
-                # normal LXD test runs).  This is not true of this test: it
-                # can't run in our usual xenial LXD VM test run, and it may not
-                # run anywhere else.  A failure flags up this discrepancy.
-                pytest.fail(XENIAL_LXD_VM_EXEC_MSG)
         launch_kwargs["execute_via_ssh"] = False
     local_launch_kwargs = {}
     if lxd_setup is not None:
@@ -276,21 +255,21 @@ def _client(request, fixture_utils, session_cloud: IntegrationCloud):
         _collect_logs(instance, request.node.nodeid, test_failed)
 
 
-@pytest.yield_fixture
+@pytest.fixture
 def client(request, fixture_utils, session_cloud, setup_image):
     """Provide a client that runs for every test."""
     with _client(request, fixture_utils, session_cloud) as client:
         yield client
 
 
-@pytest.yield_fixture(scope="module")
+@pytest.fixture(scope="module")
 def module_client(request, fixture_utils, session_cloud, setup_image):
     """Provide a client that runs once per module."""
     with _client(request, fixture_utils, session_cloud) as client:
         yield client
 
 
-@pytest.yield_fixture(scope="class")
+@pytest.fixture(scope="class")
 def class_client(request, fixture_utils, session_cloud, setup_image):
     """Provide a client that runs once per class."""
     with _client(request, fixture_utils, session_cloud) as client:
diff --git a/tests/integration_tests/datasources/test_lxd_discovery.py b/tests/integration_tests/datasources/test_lxd_discovery.py
index da010813..eb2a4cf2 100644
--- a/tests/integration_tests/datasources/test_lxd_discovery.py
+++ b/tests/integration_tests/datasources/test_lxd_discovery.py
@@ -59,13 +59,9 @@ def test_lxd_datasource_discovery(client: IntegrationInstance):
     )
     if (
         client.settings.PLATFORM == "lxd_vm"
-        and ImageSpecification.from_os_image().release
-        in (
-            "xenial",
-            "bionic",
-        )
+        and ImageSpecification.from_os_image().release == "bionic"
     ):
-        # pycloudlib injects user.vendor_data for lxd_vm on bionic and xenial
+        # pycloudlib injects user.vendor_data for lxd_vm on bionic
         # to start the lxd-agent.
         # https://github.com/canonical/pycloudlib/blob/main/pycloudlib/\
         #    lxd/defaults.py#L13-L27
diff --git a/tests/integration_tests/modules/test_disk_setup.py b/tests/integration_tests/modules/test_disk_setup.py
index 22277331..8f9d5f40 100644
--- a/tests/integration_tests/modules/test_disk_setup.py
+++ b/tests/integration_tests/modules/test_disk_setup.py
@@ -20,7 +20,7 @@ def setup_and_mount_lxd_disk(instance: LXDInstance):
     )
 
 
-@pytest.yield_fixture
+@pytest.fixture
 def create_disk():
     # 640k should be enough for anybody
     subp("dd if=/dev/zero of={} bs=1k count=640".format(DISK_PATH).split())
@@ -133,10 +133,9 @@ class TestPartProbeAvailability:
         assert sdb["children"][1]["name"] == "sdb2"
         assert sdb["children"][1]["mountpoint"] == "/mnt2"
 
-    # Not bionic or xenial because the LXD agent gets in the way of us
+    # Not bionic because the LXD agent gets in the way of us
     # changing the userdata
     @pytest.mark.not_bionic
-    @pytest.mark.not_xenial
     def test_disk_setup_when_mounted(
         self, create_disk, client: IntegrationInstance
     ):
diff --git a/tests/integration_tests/modules/test_lxd_bridge.py b/tests/integration_tests/modules/test_lxd_bridge.py
index 2cb3f4f3..3292a833 100644
--- a/tests/integration_tests/modules/test_lxd_bridge.py
+++ b/tests/integration_tests/modules/test_lxd_bridge.py
@@ -33,8 +33,6 @@ class TestLxdBridge:
         """Check that the expected LXD binaries are installed"""
         assert class_client.execute(["which", binary_name]).ok
 
-    @pytest.mark.not_xenial
-    @pytest.mark.sru_2020_11
     def test_bridge(self, class_client):
         """Check that the given bridge is configured"""
         cloud_init_log = class_client.read_from_file("/var/log/cloud-init.log")
diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py
index 3d1358ce..fddff681 100644
--- a/tests/integration_tests/modules/test_users_groups.py
+++ b/tests/integration_tests/modules/test_users_groups.py
@@ -106,7 +106,6 @@ def test_sudoers_includedir(client: IntegrationInstance):
     https://github.com/canonical/cloud-init/pull/783
     """
     if ImageSpecification.from_os_image().release in [
-        "xenial",
         "bionic",
         "focal",
     ]:
diff --git a/tests/unittests/cmd/devel/test_hotplug_hook.py b/tests/unittests/cmd/devel/test_hotplug_hook.py
index 842e8dfd..5ecb5969 100644
--- a/tests/unittests/cmd/devel/test_hotplug_hook.py
+++ b/tests/unittests/cmd/devel/test_hotplug_hook.py
@@ -16,7 +16,7 @@ hotplug_args = namedtuple("hotplug_args", "udevaction, subsystem, devpath")
 FAKE_MAC = "11:22:33:44:55:66"
 
 
-@pytest.yield_fixture
+@pytest.fixture
 def mocks():
     m_init = mock.MagicMock(spec=Init)
     m_distro = mock.MagicMock(spec=Distro)
diff --git a/tests/unittests/config/test_cc_install_hotplug.py b/tests/unittests/config/test_cc_install_hotplug.py
index 3bd44aba..e67fce60 100644
--- a/tests/unittests/config/test_cc_install_hotplug.py
+++ b/tests/unittests/config/test_cc_install_hotplug.py
@@ -12,7 +12,7 @@ from cloudinit.config.cc_install_hotplug import (
 from cloudinit.event import EventScope, EventType
 
 
-@pytest.yield_fixture()
+@pytest.fixture()
 def mocks():
     m_update_enabled = mock.patch("cloudinit.stages.update_event_enabled")
     m_write = mock.patch("cloudinit.util.write_file", autospec=True)
diff --git a/tests/unittests/distros/test_networking.py b/tests/unittests/distros/test_networking.py
index 635f6901..274647cb 100644
--- a/tests/unittests/distros/test_networking.py
+++ b/tests/unittests/distros/test_networking.py
@@ -13,7 +13,7 @@ from cloudinit.distros.networking import (
 )
 
 
-@pytest.yield_fixture
+@pytest.fixture
 def generic_networking_cls():
     """Returns a direct Networking subclass which errors on /sys usage.
 
@@ -40,7 +40,7 @@ def generic_networking_cls():
         yield TestNetworking
 
 
-@pytest.yield_fixture
+@pytest.fixture
 def sys_class_net(tmpdir):
     sys_class_net_path = tmpdir.join("sys/class/net")
     sys_class_net_path.ensure_dir()
diff --git a/tests/unittests/sources/test_lxd.py b/tests/unittests/sources/test_lxd.py
index ad1508a0..e11c3746 100644
--- a/tests/unittests/sources/test_lxd.py
+++ b/tests/unittests/sources/test_lxd.py
@@ -57,7 +57,7 @@ def lxd_metadata():
     return LXD_V1_METADATA
 
 
-@pytest.yield_fixture
+@pytest.fixture
 def lxd_ds(request, paths, lxd_metadata):
     """
     Return an instantiated DataSourceLXD.
diff --git a/tests/unittests/sources/test_oracle.py b/tests/unittests/sources/test_oracle.py
index e0e79c8c..356b3738 100644
--- a/tests/unittests/sources/test_oracle.py
+++ b/tests/unittests/sources/test_oracle.py
@@ -93,7 +93,7 @@ def metadata_version():
     return 2
 
 
-@pytest.yield_fixture
+@pytest.fixture
 def oracle_ds(request, fixture_utils, paths, metadata_version):
     """
     Return an instantiated DataSourceOracle.
@@ -649,7 +649,7 @@ class TestCommon_GetDataBehaviour:
     separate class for that case.)
     """
 
-    @pytest.yield_fixture(params=[True, False])
+    @pytest.fixture(params=[True, False])
     def parameterized_oracle_ds(self, request, oracle_ds):
         """oracle_ds parameterized for iSCSI and non-iSCSI root respectively"""
         is_iscsi_root = request.param
diff --git a/tests/unittests/sources/test_vmware.py b/tests/unittests/sources/test_vmware.py
index dcdbda89..dd331349 100644
--- a/tests/unittests/sources/test_vmware.py
+++ b/tests/unittests/sources/test_vmware.py
@@ -57,7 +57,7 @@ runcmd:
 """
 
 
-@pytest.yield_fixture(autouse=True)
+@pytest.fixture(autouse=True)
 def common_patches():
     with mock.patch("cloudinit.util.platform.platform", return_value="Linux"):
         with mock.patch.multiple(
diff --git a/tests/unittests/test_features.py b/tests/unittests/test_features.py
index 141de55b..794a9654 100644
--- a/tests/unittests/test_features.py
+++ b/tests/unittests/test_features.py
@@ -12,7 +12,7 @@ import pytest
 import cloudinit
 
 
-@pytest.yield_fixture()
+@pytest.fixture()
 def create_override(request):
     """
     Create a feature overrides file and do some module wizardry to make
diff --git a/tests/unittests/test_net_activators.py b/tests/unittests/test_net_activators.py
index 0e3ab43f..3c29e2f7 100644
--- a/tests/unittests/test_net_activators.py
+++ b/tests/unittests/test_net_activators.py
@@ -39,7 +39,7 @@ NETPLAN_CALL_LIST = [
 ]
 
 
-@pytest.yield_fixture
+@pytest.fixture
 def available_mocks():
     mocks = namedtuple("Mocks", "m_which, m_file")
     with patch("cloudinit.subp.which", return_value=True) as m_which:
@@ -47,7 +47,7 @@ def available_mocks():
             yield mocks(m_which, m_file)
 
 
-@pytest.yield_fixture
+@pytest.fixture
 def unavailable_mocks():
     mocks = namedtuple("Mocks", "m_which, m_file")
     with patch("cloudinit.subp.which", return_value=False) as m_which:
diff --git a/tests/unittests/test_stages.py b/tests/unittests/test_stages.py
index be1a0787..3214410b 100644
--- a/tests/unittests/test_stages.py
+++ b/tests/unittests/test_stages.py
@@ -512,7 +512,7 @@ class TestInit_InitializeFilesystem:
     TODO: Expand these tests to cover all of _initialize_filesystem's behavior.
     """
 
-    @pytest.yield_fixture
+    @pytest.fixture
     def init(self, paths):
         """A fixture which yields a stages.Init instance with paths and cfg set
 
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index e2bfe9d2..3765511b 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -1127,7 +1127,7 @@ class TestMountCb:
     TODO: Test the if/else branch that actually performs the mounting operation
     """
 
-    @pytest.yield_fixture
+    @pytest.fixture
     def already_mounted_device_and_mountdict(self):
         """Mock an already-mounted device, and yield (device, mount dict)"""
         device = "/dev/fake0"