Integration test upgrades for the 21.3-1 SRU (#1001)

* Update test_combined.py to allow either valid LXD subplatform
* Split jinja templated tests into separate module as they can be more
  fragile
* Move checks for warnings and tracebacks into dedicated utility
  function. This allows us to work around persistent and expected
  tracebacks/warnings on particular clouds.
* Update test_upgrade.py to allow either valid Azure datasource.
  /var/lib/waagent or a mounted device are both valid.
* Add specificity to test_ntp_servers.py
  Clouds will often specify their own ntp servers in the ntp
  configuration files, so make the tests manually specify their own.
* Account for additional keys on system in test_ssh_keysfiles.py
* Update tests to account for invalid cache
  test_user_events.py and test_version_change.py both have tests that
  assume we will have valid ds cache when rebooting.
  In test_user_events.py, subsequent boots should block applying
  network on boot if boot event is denied. However, if the cache is
  invalid, it is valid to apply networking config that boot.
  In test_version_change.py no cache found won't trigger the expected
  debug log. Additionally, the pickle used for that test on an older
  release triggered an unexpected issue that took a different error
  path.
* Ignore bionic in hotplug tests (LP: #1942247)
  On Bionic, we traceback when attempting to detect the hotplugged
  device in the updated metadata. This is because Bionic is
  specifically configured not to provide network metadata.
  See LP: #1942247 for more details.
* Fix date used in test_final_message.
  In test_final_message, we ensured the variable substitution works as
  expected. For $timestamp, we compared against the current date. It's
  possible for the host date to be massively different from the client
  date, so obtain date on client rather than host.
* Remove module success from lp1813396 test. Module may fail
  unrelatedly (in this case apt-get update is failing), but the test
  should still pass.
* Skip testing events if network is disabled
* Ensure we install expected version of cloud-init
  As part of test setup, we can install cloud-init from various
  sources, including PROPOSED, PPAs, etc. We were never checking that
  this install completes successfully, and on OCI, it wasn't
  completing successfully because of apt locking issues. Code has
  been updated to retry, and then fail loudly if we can't complete the
  install.
* Remove ubuntu-azure-fips metapkg which mandates FIPS-flavour kernel
  In test_lp1835584.py
* Update test_user_events.py to account for Azure behavior
  since Azure has a separate service to clear the pickled metadata
  every boot
* Change failure to warning in test_upgrade.py if initial boot errors
  If there's already a pre-existing cause for warnings or tracebacks,
  that shouldn't cause the new version to fail.
* Add retry to test_random_passwords_emitted_to_serial_console
  It's possible we haven't retrieved the entire log when the call returns,
  so retry a few times if the output isn't empty.

6 files changed, 14 insertions, 8 deletions

diff --git a/tests/integration_tests/bugs/test_gh632.py b/tests/integration_tests/bugs/test_gh632.py
index 3c1f9347..f3702a2e 100644
--- a/tests/integration_tests/bugs/test_gh632.py
+++ b/tests/integration_tests/bugs/test_gh632.py
@@ -3,16 +3,15 @@
 Verify that if cloud-init is using DataSourceRbxCloud, there is
 no traceback if the metadata disk cannot be found.
 """
-
 import pytest
 
 from tests.integration_tests.instances import IntegrationInstance
+from tests.integration_tests.util import verify_clean_log
 
 
 # With some datasource hacking, we can run this on a NoCloud instance
 @pytest.mark.lxd_container
 @pytest.mark.lxd_vm
-@pytest.mark.sru_2020_11
 def test_datasource_rbx_no_stacktrace(client: IntegrationInstance):
     client.write_to_file(
         '/etc/cloud/cloud.cfg.d/90_dpkg.cfg',
@@ -26,8 +25,7 @@ def test_datasource_rbx_no_stacktrace(client: IntegrationInstance):
     client.restart()
 
     log = client.read_from_file('/var/log/cloud-init.log')
-    assert 'WARNING' not in log
-    assert 'Traceback' not in log
+    verify_clean_log(log)
     assert 'Failed to load metadata and userdata' not in log
     assert ("Getting data from <class 'cloudinit.sources.DataSourceRbxCloud."
             "DataSourceRbxCloud'> failed") not in log
diff --git a/tests/integration_tests/bugs/test_gh868.py b/tests/integration_tests/bugs/test_gh868.py
index 838efca6..73c03451 100644
--- a/tests/integration_tests/bugs/test_gh868.py
+++ b/tests/integration_tests/bugs/test_gh868.py
@@ -1,6 +1,8 @@
 """Ensure no Traceback when 'chef_license' is set"""
 import pytest
+
 from tests.integration_tests.instances import IntegrationInstance
+from tests.integration_tests.util import verify_clean_log
 
 
 USERDATA = """\
@@ -17,4 +19,4 @@ chef:
 @pytest.mark.user_data(USERDATA)
 def test_chef_license(client: IntegrationInstance):
     log = client.read_from_file('/var/log/cloud-init.log')
-    assert 'Traceback' not in log
+    verify_clean_log(log)
diff --git a/tests/integration_tests/bugs/test_lp1813396.py b/tests/integration_tests/bugs/test_lp1813396.py
index 68b96b1d..27d41c2b 100644
--- a/tests/integration_tests/bugs/test_lp1813396.py
+++ b/tests/integration_tests/bugs/test_lp1813396.py
@@ -29,6 +29,5 @@ def test_gpg_no_tty(client: IntegrationInstance):
         "'--keyserver=keyserver.ubuntu.com', '--recv-keys', 'E4D304DF'] "
         "with allowed return codes [0] (shell=False, capture=True)",
         "Imported key 'E4D304DF' from keyserver 'keyserver.ubuntu.com'",
-        "finish: modules-config/config-apt-configure: SUCCESS",
     ]
     verify_ordered_items_in_text(to_verify, log)
diff --git a/tests/integration_tests/bugs/test_lp1835584.py b/tests/integration_tests/bugs/test_lp1835584.py
index 660d2a2a..732f2179 100644
--- a/tests/integration_tests/bugs/test_lp1835584.py
+++ b/tests/integration_tests/bugs/test_lp1835584.py
@@ -59,6 +59,9 @@ def _check_iid_insensitive_across_kernel_upgrade(
     result = instance.execute("apt-get install linux-azure --assume-yes")
     if not result.ok:
         pytest.fail("Unable to install linux-azure kernel: {}".format(result))
+    # Remove ubuntu-azure-fips metapkg which mandates FIPS-flavour kernel
+    result = instance.execute("ua disable fips --assume-yes")
+    assert result.ok, "Unable to disable fips: {}".format(result)
     instance.restart()
     new_kernel = instance.execute("uname -r").strip()
     assert orig_kernel != new_kernel
diff --git a/tests/integration_tests/bugs/test_lp1886531.py b/tests/integration_tests/bugs/test_lp1886531.py
index 058ea8bb..6dd61222 100644
--- a/tests/integration_tests/bugs/test_lp1886531.py
+++ b/tests/integration_tests/bugs/test_lp1886531.py
@@ -11,6 +11,8 @@ https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/1886531
 """
 import pytest
 
+from tests.integration_tests.util import verify_clean_log
+
 
 USER_DATA = """\
 #cloud-config
@@ -24,4 +26,4 @@ class TestLp1886531:
     @pytest.mark.user_data(USER_DATA)
     def test_lp1886531(self, client):
         log_content = client.read_from_file("/var/log/cloud-init.log")
-        assert "WARNING" not in log_content
+        verify_clean_log(log_content)
diff --git a/tests/integration_tests/bugs/test_lp1898997.py b/tests/integration_tests/bugs/test_lp1898997.py
index bde93d06..909bc690 100644
--- a/tests/integration_tests/bugs/test_lp1898997.py
+++ b/tests/integration_tests/bugs/test_lp1898997.py
@@ -10,7 +10,9 @@ network configuration, and confirms that the bridge can be used to ping the
 default gateway.
 """
 import pytest
+
 from tests.integration_tests import random_mac_address
+from tests.integration_tests.util import verify_clean_log
 
 MAC_ADDRESS = random_mac_address()
 
@@ -59,7 +61,7 @@ class TestInterfaceListingWithOpenvSwitch:
         cloudinit_output = client.read_from_file("/var/log/cloud-init.log")
 
         # Confirm that the network configuration was applied successfully
-        assert "WARN" not in cloudinit_output
+        verify_clean_log(cloudinit_output)
         # Confirm that the applied network config created the OVS bridge
         assert "ovs-br" in client.execute("ip addr")