From 6c4e87bf336073183f8ae8964366d574c7ee4823 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 3 Dec 2020 13:17:55 -0500 Subject: integration_tests: introduce skipping of tests by OS (#702) This introduces an optional, more complex OS_IMAGE format (`::::`) which allows the specification of the OS/OS release which the given image ID corresponds to. This information is used to skip tests which do not apply to the image. This commit is comprised of the following discrete changes: * introduce the IntegrationImage class, to handle parsing and storing the new OS_IMAGE format * support inferring the OS and OS release of Ubuntu series, so that we can continue to set OS_IMAGE to just a series name and have test skipping work * add documentation on Image Selection to integration_tests.rst * introduce the actual skipping behaviour based on OS marks * apply the `ubuntu` mark to all tests that should be skipped on non-Ubuntu operating systems --- tests/integration_tests/modules/test_ssh_import_id.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'tests/integration_tests/modules/test_ssh_import_id.py') diff --git a/tests/integration_tests/modules/test_ssh_import_id.py b/tests/integration_tests/modules/test_ssh_import_id.py index 45d37d6c..3db573b5 100644 --- a/tests/integration_tests/modules/test_ssh_import_id.py +++ b/tests/integration_tests/modules/test_ssh_import_id.py @@ -3,6 +3,10 @@ This test specifies ssh keys to be imported by the ``ssh_import_id`` module and then checks that if the ssh keys were successfully imported. +TODO: +* This test assumes that SSH keys will be imported into the /home/ubuntu; this + will need modification to run on other OSes. + (This is ported from ``tests/cloud_tests/testcases/modules/ssh_import_id.yaml``.)""" @@ -18,6 +22,7 @@ ssh_import_id: @pytest.mark.ci +@pytest.mark.ubuntu class TestSshImportId: @pytest.mark.user_data(USER_DATA) -- cgit v1.2.3 From 824977bd58bae601600682f134bfec00b0c69bbd Mon Sep 17 00:00:00 2001 From: James Falcon Date: Thu, 29 Jul 2021 12:29:46 -0500 Subject: testing: fix test_ssh_import_id.py (#954) test_ssh_import_id.py occassionally fails because cloud-init finishes before the keys have been fully imported. A retry has been added to the test. --- .../modules/test_ssh_import_id.py | 6 +++++ tests/integration_tests/util.py | 30 ++++++++++++++++++++++ 2 files changed, 36 insertions(+) (limited to 'tests/integration_tests/modules/test_ssh_import_id.py') diff --git a/tests/integration_tests/modules/test_ssh_import_id.py b/tests/integration_tests/modules/test_ssh_import_id.py index 3db573b5..b90fe95f 100644 --- a/tests/integration_tests/modules/test_ssh_import_id.py +++ b/tests/integration_tests/modules/test_ssh_import_id.py @@ -12,6 +12,7 @@ TODO: import pytest +from tests.integration_tests.util import retry USER_DATA = """\ #cloud-config @@ -26,6 +27,11 @@ ssh_import_id: class TestSshImportId: @pytest.mark.user_data(USER_DATA) + # Retry is needed here because ssh import id is one of the last modules + # run, and it fires off a web request, then continues with the rest of + # cloud-init. It is possible cloud-init's status is "done" before the + # id's have been fully imported. + @retry(tries=30, delay=1) def test_ssh_import_id(self, client): ssh_output = client.read_from_file( "/home/ubuntu/.ssh/authorized_keys") diff --git a/tests/integration_tests/util.py b/tests/integration_tests/util.py index ce62ffc8..80430eab 100644 --- a/tests/integration_tests/util.py +++ b/tests/integration_tests/util.py @@ -1,3 +1,4 @@ +import functools import logging import multiprocessing import os @@ -64,3 +65,32 @@ def get_test_rsa_keypair(key_name: str = 'test1') -> key_pair: with private_key_path.open() as private_file: private_key = private_file.read() return key_pair(public_key, private_key) + + +def retry(*, tries: int = 30, delay: int = 1): + """Decorator for retries. + + Retry a function until code no longer raises an exception or + max tries is reached. + + Example: + @retry(tries=5, delay=1) + def try_something_that_may_not_be_ready(): + ... + """ + def _retry(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + last_error = None + for _ in range(tries): + try: + func(*args, **kwargs) + break + except Exception as e: + last_error = e + time.sleep(delay) + else: + if last_error: + raise last_error + return wrapper + return _retry -- cgit v1.2.3 From 8c52bb3fc530742fce50f7f1061a24f3c453ef94 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Tue, 16 Nov 2021 18:04:57 -0600 Subject: integration_test: Speed up CI run time (#1111) Move more tests into test_combined.py and remove the CI mark from module tests that aren't updated often or don't represent core functionality. --- tests/integration_tests/modules/test_apt.py | 1 - tests/integration_tests/modules/test_combined.py | 68 ++++++++++++++++++++++ .../modules/test_command_output.py | 1 - .../integration_tests/modules/test_ntp_servers.py | 3 - .../modules/test_seed_random_data.py | 30 ---------- tests/integration_tests/modules/test_snap.py | 30 ---------- .../modules/test_ssh_import_id.py | 40 ------------- tests/integration_tests/modules/test_timezone.py | 25 -------- 8 files changed, 68 insertions(+), 130 deletions(-) delete mode 100644 tests/integration_tests/modules/test_seed_random_data.py delete mode 100644 tests/integration_tests/modules/test_snap.py delete mode 100644 tests/integration_tests/modules/test_ssh_import_id.py delete mode 100644 tests/integration_tests/modules/test_timezone.py (limited to 'tests/integration_tests/modules/test_ssh_import_id.py') diff --git a/tests/integration_tests/modules/test_apt.py b/tests/integration_tests/modules/test_apt.py index 2de3e202..f5f6c813 100644 --- a/tests/integration_tests/modules/test_apt.py +++ b/tests/integration_tests/modules/test_apt.py @@ -97,7 +97,6 @@ TEST_KEY = "1FF0 D853 5EF7 E719 E5C8 1B9C 083D 06FB E4D3 04DF" TEST_SIGNED_BY_KEY = "A2EB 2DEC 0BD7 519B 7B38 BE38 376A 290E C806 8B11" -@pytest.mark.ci @pytest.mark.ubuntu @pytest.mark.user_data(USER_DATA) class TestApt: diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 57c02f47..2635d41a 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -12,6 +12,7 @@ import re from tests.integration_tests.clouds import ImageSpecification from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.util import ( + retry, verify_clean_log, verify_ordered_items_in_text, ) @@ -33,6 +34,11 @@ locale: en_GB.UTF-8 locale_configfile: /etc/default/locale ntp: servers: ['ntp.ubuntu.com'] +package_update: true +random_seed: + data: 'MYUb34023nD:LFDK10913jk;dfnk:Df' + encoding: raw + file: /root/seed rsyslog: configs: - "*.* @@127.0.0.1" @@ -48,6 +54,14 @@ rsyslog: runcmd: - echo 'hello world' > /var/tmp/runcmd_output - logger "My test log" +snap: + squashfuse_in_container: true + commands: + - snap install hello-world +ssh_import_id: + - gh:powersj + - lp:smoser +timezone: US/Aleutian """ @@ -115,6 +129,20 @@ class TestCombined: 'en_US.UTF-8' ], locale_gen) + def test_random_seed_data(self, class_client: IntegrationInstance): + """Integration test for the random seed module. + + This test specifies a command to be executed by the ``seed_random`` + module, by providing a different data to be used as seed data. We will + then check if that seed data was actually used. + """ + client = class_client + + # Only read the first 31 characters, because the rest could be + # binary data + result = client.execute("head -c 31 < /root/seed") + assert result.startswith("MYUb34023nD:LFDK10913jk;dfnk:Df") + def test_rsyslog(self, class_client: IntegrationInstance): """Test rsyslog is configured correctly.""" client = class_client @@ -125,6 +153,46 @@ class TestCombined: client = class_client assert 'hello world' == client.read_from_file('/var/tmp/runcmd_output') + @retry(tries=30, delay=1) + def test_ssh_import_id(self, class_client: IntegrationInstance): + """Integration test for the ssh_import_id module. + + This test specifies ssh keys to be imported by the ``ssh_import_id`` + module and then checks that if the ssh keys were successfully imported. + + TODO: + * This test assumes that SSH keys will be imported into the + /home/ubuntu; this will need modification to run on other OSes. + """ + client = class_client + ssh_output = client.read_from_file( + "/home/ubuntu/.ssh/authorized_keys") + + assert '# ssh-import-id gh:powersj' in ssh_output + assert '# ssh-import-id lp:smoser' in ssh_output + + def test_snap(self, class_client: IntegrationInstance): + """Integration test for the snap module. + + This test specifies a command to be executed by the ``snap`` module + and then checks that if that command was executed during boot. + """ + client = class_client + snap_output = client.execute("snap list") + assert "core " in snap_output + assert "hello-world " in snap_output + + def test_timezone(self, class_client: IntegrationInstance): + """Integration test for the timezone module. + + This test specifies a timezone to be used by the ``timezone`` module + and then checks that if that timezone was respected during boot. + """ + client = class_client + timezone_output = client.execute( + 'date "+%Z" --date="Thu, 03 Nov 2016 00:47:00 -0400"') + assert timezone_output.strip() == "HDT" + def test_no_problems(self, class_client: IntegrationInstance): """Test no errors, warnings, or tracebacks""" client = class_client diff --git a/tests/integration_tests/modules/test_command_output.py b/tests/integration_tests/modules/test_command_output.py index 15033642..8429873f 100644 --- a/tests/integration_tests/modules/test_command_output.py +++ b/tests/integration_tests/modules/test_command_output.py @@ -16,7 +16,6 @@ final_message: "should be last line in cloud-init-test-output file" """ -@pytest.mark.ci @pytest.mark.user_data(USER_DATA) def test_runcmd(client: IntegrationInstance): log = client.read_from_file('/var/log/cloud-init-test-output') diff --git a/tests/integration_tests/modules/test_ntp_servers.py b/tests/integration_tests/modules/test_ntp_servers.py index 59241faa..c777a641 100644 --- a/tests/integration_tests/modules/test_ntp_servers.py +++ b/tests/integration_tests/modules/test_ntp_servers.py @@ -31,7 +31,6 @@ EXPECTED_SERVERS = yaml.safe_load(USER_DATA)["ntp"]["servers"] EXPECTED_POOLS = yaml.safe_load(USER_DATA)["ntp"]["pools"] -@pytest.mark.ci @pytest.mark.user_data(USER_DATA) class TestNtpServers: @@ -83,7 +82,6 @@ ntp: """ -@pytest.mark.ci @pytest.mark.user_data(CHRONY_DATA) def test_chrony(client: IntegrationInstance): if client.execute('test -f /etc/chrony.conf').ok: @@ -104,7 +102,6 @@ ntp: """ -@pytest.mark.ci @pytest.mark.user_data(TIMESYNCD_DATA) def test_timesyncd(client: IntegrationInstance): contents = client.read_from_file( diff --git a/tests/integration_tests/modules/test_seed_random_data.py b/tests/integration_tests/modules/test_seed_random_data.py deleted file mode 100644 index 94e982e0..00000000 --- a/tests/integration_tests/modules/test_seed_random_data.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Integration test for the random seed module. - -This test specifies a command to be executed by the ``seed_random`` module, by -providing a different data to be used as seed data. We will then check -if that seed data was actually used. - -(This is ported from -``tests/cloud_tests/testcases/modules/seed_random_data.yaml``.)""" - -import pytest - - -USER_DATA = """\ -#cloud-config -random_seed: - data: 'MYUb34023nD:LFDK10913jk;dfnk:Df' - encoding: raw - file: /root/seed -""" - - -@pytest.mark.ci -class TestSeedRandomData: - - @pytest.mark.user_data(USER_DATA) - def test_seed_random_data(self, client): - # Only read the first 31 characters, because the rest could be - # binary data - result = client.execute("head -c 31 < /root/seed") - assert result.startswith("MYUb34023nD:LFDK10913jk;dfnk:Df") diff --git a/tests/integration_tests/modules/test_snap.py b/tests/integration_tests/modules/test_snap.py deleted file mode 100644 index 652efa68..00000000 --- a/tests/integration_tests/modules/test_snap.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Integration test for the snap module. - -This test specifies a command to be executed by the ``snap`` module -and then checks that if that command was executed during boot. - -(This is ported from -``tests/cloud_tests/testcases/modules/snap.yaml``.)""" - -import pytest - - -USER_DATA = """\ -#cloud-config -package_update: true -snap: - squashfuse_in_container: true - commands: - - snap install hello-world -""" - - -@pytest.mark.ci -@pytest.mark.ubuntu -class TestSnap: - - @pytest.mark.user_data(USER_DATA) - def test_snap(self, client): - snap_output = client.execute("snap list") - assert "core " in snap_output - assert "hello-world " in snap_output diff --git a/tests/integration_tests/modules/test_ssh_import_id.py b/tests/integration_tests/modules/test_ssh_import_id.py deleted file mode 100644 index b90fe95f..00000000 --- a/tests/integration_tests/modules/test_ssh_import_id.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Integration test for the ssh_import_id module. - -This test specifies ssh keys to be imported by the ``ssh_import_id`` module -and then checks that if the ssh keys were successfully imported. - -TODO: -* This test assumes that SSH keys will be imported into the /home/ubuntu; this - will need modification to run on other OSes. - -(This is ported from -``tests/cloud_tests/testcases/modules/ssh_import_id.yaml``.)""" - -import pytest - -from tests.integration_tests.util import retry - -USER_DATA = """\ -#cloud-config -ssh_import_id: - - gh:powersj - - lp:smoser -""" - - -@pytest.mark.ci -@pytest.mark.ubuntu -class TestSshImportId: - - @pytest.mark.user_data(USER_DATA) - # Retry is needed here because ssh import id is one of the last modules - # run, and it fires off a web request, then continues with the rest of - # cloud-init. It is possible cloud-init's status is "done" before the - # id's have been fully imported. - @retry(tries=30, delay=1) - def test_ssh_import_id(self, client): - ssh_output = client.read_from_file( - "/home/ubuntu/.ssh/authorized_keys") - - assert '# ssh-import-id gh:powersj' in ssh_output - assert '# ssh-import-id lp:smoser' in ssh_output diff --git a/tests/integration_tests/modules/test_timezone.py b/tests/integration_tests/modules/test_timezone.py deleted file mode 100644 index 111d53f7..00000000 --- a/tests/integration_tests/modules/test_timezone.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Integration test for the timezone module. - -This test specifies a timezone to be used by the ``timezone`` module -and then checks that if that timezone was respected during boot. - -(This is ported from -``tests/cloud_tests/testcases/modules/timezone.yaml``.)""" - -import pytest - - -USER_DATA = """\ -#cloud-config -timezone: US/Aleutian -""" - - -@pytest.mark.ci -class TestTimezone: - - @pytest.mark.user_data(USER_DATA) - def test_timezone(self, client): - timezone_output = client.execute( - 'date "+%Z" --date="Thu, 03 Nov 2016 00:47:00 -0400"') - assert timezone_output.strip() == "HDT" -- cgit v1.2.3