Add integration test for power_state_change module (#717)

Also introduce the `unstable` mark, to allow us to land tests which
run inconsistently (such as this one).

6 files changed, 119 insertions, 7 deletions

diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py
index ea42b6d5..9f6a6380 100644
--- a/tests/integration_tests/clouds.py
+++ b/tests/integration_tests/clouds.py
@@ -124,11 +124,12 @@ class IntegrationCloud(ABC):
 
     def _perform_launch(self, launch_kwargs):
         pycloudlib_instance = self.cloud_instance.launch(**launch_kwargs)
-        pycloudlib_instance.wait(raise_on_cloudinit_failure=False)
         return pycloudlib_instance
 
-    def launch(self, user_data=None, launch_kwargs=None,
+    def launch(self, user_data=None, launch_kwargs=None, wait=True,
                settings=integration_settings):
+        if launch_kwargs is None:
+            launch_kwargs = {}
         if self.settings.EXISTING_INSTANCE_ID:
             log.info(
                 'Not launching instance due to EXISTING_INSTANCE_ID. '
@@ -137,13 +138,15 @@ class IntegrationCloud(ABC):
                 self.settings.EXISTING_INSTANCE_ID
             )
             return
+        if 'wait' in launch_kwargs:
+            raise Exception("Specify 'wait' directly to launch, "
+                            "not in 'launch_kwargs'")
         kwargs = {
             'image_id': self.image_id,
             'user_data': user_data,
             'wait': False,
         }
-        if launch_kwargs:
-            kwargs.update(launch_kwargs)
+        kwargs.update(launch_kwargs)
         log.info(
             "Launching instance with launch_kwargs:\n{}".format(
                 "\n".join("{}={}".format(*item) for item in kwargs.items())
@@ -151,7 +154,8 @@ class IntegrationCloud(ABC):
         )
 
         pycloudlib_instance = self._perform_launch(kwargs)
-
+        if wait:
+            pycloudlib_instance.wait(raise_on_cloudinit_failure=False)
         log.info('Launched instance: %s', pycloudlib_instance)
         return self.get_instance(pycloudlib_instance, settings)
 
@@ -275,7 +279,6 @@ class _LxdIntegrationCloud(IntegrationCloud):
         if self.settings.CLOUD_INIT_SOURCE == 'IN_PLACE':
             self._mount_source(pycloudlib_instance)
         pycloudlib_instance.start(wait=False)
-        pycloudlib_instance.wait(raise_on_cloudinit_failure=False)
         return pycloudlib_instance
 
 
diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py
index 160fc085..53ca5fb5 100644
--- a/tests/integration_tests/conftest.py
+++ b/tests/integration_tests/conftest.py
@@ -71,6 +71,8 @@ def pytest_runtest_setup(item):
     supported_os_set = set(os_list).intersection(test_marks)
     if current_os and supported_os_set and current_os not in supported_os_set:
         pytest.skip("Cannot run on OS {}".format(current_os))
+    if 'unstable' in test_marks and not integration_settings.RUN_UNSTABLE:
+        pytest.skip('Test marked unstable. Manually remove mark to run it')
 
 
 # disable_subp_usage is defined at a higher level, but we don't
@@ -176,7 +178,7 @@ def _collect_logs(instance: IntegrationInstance, node_id: str,
 
 
 @contextmanager
-def _client(request, fixture_utils, session_cloud):
+def _client(request, fixture_utils, session_cloud: IntegrationCloud):
     """Fixture implementation for the client fixtures.
 
     Launch the dynamic IntegrationClient instance using any provided
diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py
index ad6d453a..9948d479 100644
--- a/tests/integration_tests/integration_settings.py
+++ b/tests/integration_tests/integration_settings.py
@@ -9,6 +9,8 @@ import os
 KEEP_INSTANCE = False
 # Keep snapshot image (mostly for debugging) when test is finished
 KEEP_IMAGE = False
+# Run tests marked as unstable. Expect failures and dragons.
+RUN_UNSTABLE = False
 
 # One of:
 #  lxd_container
diff --git a/tests/integration_tests/log_utils.py b/tests/integration_tests/log_utils.py
new file mode 100644
index 00000000..fa807389
--- /dev/null
+++ b/tests/integration_tests/log_utils.py
@@ -0,0 +1,13 @@
+def ordered_items_in_text(to_verify: list, text: str) -> bool:
+    """Return if all items in list appear in order in text.
+
+    Examples:
+      ordered_items_in_text(['a', '1'], 'ab1')  # Returns True
+      ordered_items_in_text(['1', 'a'], 'ab1')  # Returns False
+    """
+    index = 0
+    for item in to_verify:
+        index = text[index:].find(item)
+        if index < 0:
+            return False
+    return True
diff --git a/tests/integration_tests/modules/test_power_state_change.py b/tests/integration_tests/modules/test_power_state_change.py
new file mode 100644
index 00000000..60e0e583
--- /dev/null
+++ b/tests/integration_tests/modules/test_power_state_change.py
@@ -0,0 +1,91 @@
+"""Integration test of the cc_power_state_change module.
+
+Test that the power state config options work as expected.
+"""
+
+import time
+
+import pytest
+
+from tests.integration_tests.clouds import IntegrationCloud
+from tests.integration_tests.instances import IntegrationInstance
+from tests.integration_tests.log_utils import ordered_items_in_text
+
+USER_DATA = """\
+#cloud-config
+power_state:
+  delay: {delay}
+  mode: {mode}
+  message: msg
+  timeout: {timeout}
+  condition: {condition}
+"""
+
+
+def _detect_reboot(instance: IntegrationInstance):
+    # We'll wait for instance up here, but we don't know if we're
+    # detecting the first boot or second boot, so we also check
+    # the logs to ensure we've booted twice. If the logs show we've
+    # only booted once, wait until we've booted twice
+    instance.instance.wait(raise_on_cloudinit_failure=False)
+    for _ in range(600):
+        try:
+            log = instance.read_from_file('/var/log/cloud-init.log')
+            boot_count = log.count("running 'init-local'")
+            if boot_count == 1:
+                instance.instance.wait(raise_on_cloudinit_failure=False)
+            elif boot_count > 1:
+                break
+        except Exception:
+            pass
+        time.sleep(1)
+    else:
+        raise Exception('Could not detect reboot')
+
+
+def _can_connect(instance):
+    return instance.execute('true').ok
+
+
+# This test is marked unstable because even though it should be able to
+# run anywhere, I can only get it to run in an lxd container, and even then
+# occasionally some timing issues will crop up.
+@pytest.mark.unstable
+@pytest.mark.sru_2020_11
+@pytest.mark.ubuntu
+@pytest.mark.lxd_container
+class TestPowerChange:
+    @pytest.mark.parametrize('mode,delay,timeout,expected', [
+        ('poweroff', 'now', '10', 'will execute: shutdown -P now msg'),
+        ('reboot', 'now', '0', 'will execute: shutdown -r now msg'),
+        ('halt', '+1', '0', 'will execute: shutdown -H +1 msg'),
+    ])
+    def test_poweroff(self, session_cloud: IntegrationCloud,
+                      mode, delay, timeout, expected):
+        with session_cloud.launch(
+            user_data=USER_DATA.format(
+                delay=delay, mode=mode, timeout=timeout, condition='true'),
+            wait=False
+        ) as instance:
+            if mode == 'reboot':
+                _detect_reboot(instance)
+            else:
+                instance.instance.wait_for_stop()
+                instance.instance.start(wait=True)
+            log = instance.read_from_file('/var/log/cloud-init.log')
+            assert _can_connect(instance)
+        lines_to_check = [
+            'Running module power-state-change',
+            expected,
+            "running 'init-local'",
+            'config-power-state-change already ran',
+        ]
+        assert ordered_items_in_text(lines_to_check, log), (
+            'Expected data not in logs')
+
+    @pytest.mark.user_data(USER_DATA.format(delay='0', mode='poweroff',
+                                            timeout='0', condition='false'))
+    def test_poweroff_false_condition(self, client: IntegrationInstance):
+        log = client.read_from_file('/var/log/cloud-init.log')
+        assert _can_connect(client)
+        assert 'Condition was false. Will not perform state change' in log
diff --git a/tox.ini b/tox.ini
index df1deb6f..1841247b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -180,3 +180,4 @@ markers =
     instance_name: the name to be used for the test instance
     sru_2020_11: test is part of the 2020/11 SRU verification
     ubuntu: this test should run on Ubuntu
+    unstable: skip this test because it is flakey