From 268fefa309c20181f18ce1081e26aa90cc0b2f85 Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Fri, 16 Oct 2020 10:22:50 -0400
Subject: integration_tests: implement citest tests run in Travis (#605)

Specifically:
* `apt_configure_sources_list`
* `ntp_servers`
* `set_password_list`
* `users_groups`

Although not currently run in Travis, `set_password_list_string` was
ported over alongside `set_password_list` (as `test_set_password`).
---
 .../integration_tests/modules/test_set_password.py | 149 +++++++++++++++++++++
 1 file changed, 149 insertions(+)
 create mode 100644 tests/integration_tests/modules/test_set_password.py

(limited to 'tests/integration_tests/modules/test_set_password.py')

diff --git a/tests/integration_tests/modules/test_set_password.py b/tests/integration_tests/modules/test_set_password.py
new file mode 100644
index 00000000..ae6fdefc
--- /dev/null
+++ b/tests/integration_tests/modules/test_set_password.py
@@ -0,0 +1,149 @@
+"""Integration test for the set_password module.
+
+This test specifies a combination of user/password pairs, and ensures that the
+system has the correct passwords set.
+
+There are two tests run here: one tests chpasswd's list being a YAML list, the
+other tests chpasswd's list being a string.  Both expect the same results, so
+they use a mixin to share their test definitions, because we can (of course)
+only specify one user-data per instance.
+"""
+import crypt
+
+import pytest
+import yaml
+
+
+COMMON_USER_DATA = """\
+#cloud-config
+ssh_pwauth: yes
+users:
+  - default
+  - name: tom
+    # md5 gotomgo
+    passwd: "$1$S7$tT1BEDIYrczeryDQJfdPe0"
+    lock_passwd: false
+  - name: dick
+    # md5 gocubsgo
+    passwd: "$1$ssisyfpf$YqvuJLfrrW6Cg/l53Pi1n1"
+    lock_passwd: false
+  - name: harry
+    # sha512 goharrygo
+    passwd: "$6$LF$9Z2p6rWK6TNC1DC6393ec0As.18KRAvKDbfsGJEdWN3sRQRwpdfoh37EQ3y\
+Uh69tP4GSrGW5XKHxMLiKowJgm/"
+    lock_passwd: false
+  - name: jane
+    # sha256 gojanego
+    passwd: "$5$iW$XsxmWCdpwIW8Yhv.Jn/R3uk6A4UaicfW5Xp7C9p9pg."
+    lock_passwd: false
+  - name: "mikey"
+    lock_passwd: false
+"""
+
+LIST_USER_DATA = COMMON_USER_DATA + """
+chpasswd:
+  list:
+    - tom:mypassword123!
+    - dick:RANDOM
+    - harry:RANDOM
+    - mikey:$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89
+"""
+
+STRING_USER_DATA = COMMON_USER_DATA + """
+chpasswd:
+    list: |
+      tom:mypassword123!
+      dick:RANDOM
+      harry:RANDOM
+      mikey:$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89
+"""
+
+USERS_DICTS = yaml.safe_load(COMMON_USER_DATA)["users"]
+USERS_PASSWD_VALUES = {
+    user_dict["name"]: user_dict["passwd"]
+    for user_dict in USERS_DICTS
+    if "name" in user_dict and "passwd" in user_dict
+}
+
+
+class Mixin:
+    """Shared test definitions."""
+
+    def _fetch_and_parse_etc_shadow(self, class_client):
+        """Fetch /etc/shadow and parse it into Python data structures
+
+        Returns: ({user: password}, [duplicate, users])
+        """
+        shadow_content = class_client.read_from_file("/etc/shadow")
+        users = {}
+        dupes = []
+        for line in shadow_content.splitlines():
+            user, encpw = line.split(":")[0:2]
+            if user in users:
+                dupes.append(user)
+            users[user] = encpw
+        return users, dupes
+
+    def test_no_duplicate_users_in_shadow(self, class_client):
+        """Confirm that set_passwords has not added duplicate shadow entries"""
+        _, dupes = self._fetch_and_parse_etc_shadow(class_client)
+
+        assert [] == dupes
+
+    def test_password_in_users_dict_set_correctly(self, class_client):
+        """Test that the password specified in the users dict is set."""
+        shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client)
+        assert USERS_PASSWD_VALUES["jane"] == shadow_users["jane"]
+
+    def test_password_in_chpasswd_list_set_correctly(self, class_client):
+        """Test that a chpasswd password overrides one in the users dict."""
+        shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client)
+        mikey_hash = "$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89"
+        assert mikey_hash == shadow_users["mikey"]
+
+    def test_random_passwords_set_correctly(self, class_client):
+        """Test that RANDOM chpasswd entries replace users dict passwords."""
+        shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client)
+
+        # These should have been changed
+        assert shadow_users["harry"] != USERS_PASSWD_VALUES["harry"]
+        assert shadow_users["dick"] != USERS_PASSWD_VALUES["dick"]
+
+        # To random passwords
+        assert shadow_users["harry"].startswith("$")
+        assert shadow_users["dick"].startswith("$")
+
+        # Which are not the same
+        assert shadow_users["harry"] != shadow_users["dick"]
+
+    def test_explicit_password_set_correctly(self, class_client):
+        """Test that an explicitly-specified password is set correctly."""
+        shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client)
+
+        fmt_and_salt = shadow_users["tom"].rsplit("$", 1)[0]
+        expected_value = crypt.crypt("mypassword123!", fmt_and_salt)
+
+        assert expected_value == shadow_users["tom"]
+
+    def test_shadow_expected_users(self, class_client):
+        """Test that the right set of users is in /etc/shadow."""
+        shadow = class_client.read_from_file("/etc/shadow")
+        for user_dict in USERS_DICTS:
+            if "name" in user_dict:
+                assert "{}:".format(user_dict["name"]) in shadow
+
+    def test_sshd_config(self, class_client):
+        """Test that SSH password auth is enabled."""
+        sshd_config = class_client.read_from_file("/etc/ssh/sshd_config")
+        # We look for the exact line match, to avoid a commented line matching
+        assert "PasswordAuthentication yes" in sshd_config.splitlines()
+
+
+@pytest.mark.user_data(LIST_USER_DATA)
+class TestPasswordList(Mixin):
+    """Launch an instance with LIST_USER_DATA, ensure Mixin tests pass."""
+
+
+@pytest.mark.user_data(STRING_USER_DATA)
+class TestPasswordListString(Mixin):
+    """Launch an instance with STRING_USER_DATA, ensure Mixin tests pass."""
-- 
cgit v1.2.3


From cd752df6154c403e6dccaf5e797c1d4f8396f756 Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Wed, 18 Nov 2020 09:48:47 -0500
Subject: only run a subset of integration tests in CI (#672)

This introduces the "ci" mark, used to indicate a test which should run
as part of our CI integration testing run and the integration-tests-ci
tox environment, which runs only those tests. Travis has been adjusted
to use this tox environment.

(All current module tests have been marked with the "ci" mark, but the
one bug test that we have has not.)
---
 .travis.yml                                                       | 2 +-
 .../integration_tests/modules/test_apt_configure_sources_list.py  | 1 +
 tests/integration_tests/modules/test_ntp_servers.py               | 1 +
 tests/integration_tests/modules/test_runcmd.py                    | 1 +
 tests/integration_tests/modules/test_seed_random_data.py          | 1 +
 tests/integration_tests/modules/test_set_hostname.py              | 1 +
 tests/integration_tests/modules/test_set_password.py              | 2 ++
 tests/integration_tests/modules/test_snap.py                      | 1 +
 tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py | 1 +
 tests/integration_tests/modules/test_ssh_generate.py              | 1 +
 tests/integration_tests/modules/test_ssh_import_id.py             | 1 +
 tests/integration_tests/modules/test_ssh_keys_provided.py         | 1 +
 tests/integration_tests/modules/test_timezone.py                  | 1 +
 tests/integration_tests/modules/test_users_groups.py              | 1 +
 tests/integration_tests/modules/test_write_files.py               | 1 +
 tox.ini                                                           | 8 ++++++++
 16 files changed, 24 insertions(+), 1 deletion(-)

(limited to 'tests/integration_tests/modules/test_set_password.py')

diff --git a/.travis.yml b/.travis.yml
index 496c1a81..6d25f477 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -199,7 +199,7 @@ matrix:
                 fi
             # Use sudo to get a new shell where we're in the sbuild group
             - sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc'
-            - sg lxd -c 'CLOUD_INIT_IMAGE_SOURCE="$(ls *.deb)" tox -e integration-tests' &
+            - sg lxd -c 'CLOUD_INIT_IMAGE_SOURCE="$(ls *.deb)" tox -e integration-tests-ci' &
             - |
               SECONDS=0
               while [ -e /proc/$! ]; do
diff --git a/tests/integration_tests/modules/test_apt_configure_sources_list.py b/tests/integration_tests/modules/test_apt_configure_sources_list.py
index d64b3956..d2bcc61a 100644
--- a/tests/integration_tests/modules/test_apt_configure_sources_list.py
+++ b/tests/integration_tests/modules/test_apt_configure_sources_list.py
@@ -39,6 +39,7 @@ EXPECTED_REGEXES = [
 ]
 
 
+@pytest.mark.ci
 class TestAptConfigureSourcesList:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_ntp_servers.py b/tests/integration_tests/modules/test_ntp_servers.py
index 4cad8926..e72389c1 100644
--- a/tests/integration_tests/modules/test_ntp_servers.py
+++ b/tests/integration_tests/modules/test_ntp_servers.py
@@ -22,6 +22,7 @@ ntp:
 EXPECTED_SERVERS = yaml.safe_load(USER_DATA)["ntp"]["servers"]
 
 
+@pytest.mark.ci
 @pytest.mark.user_data(USER_DATA)
 class TestNtpServers:
 
diff --git a/tests/integration_tests/modules/test_runcmd.py b/tests/integration_tests/modules/test_runcmd.py
index eabe778d..50d1851e 100644
--- a/tests/integration_tests/modules/test_runcmd.py
+++ b/tests/integration_tests/modules/test_runcmd.py
@@ -16,6 +16,7 @@ runcmd:
 """
 
 
+@pytest.mark.ci
 class TestRuncmd:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_seed_random_data.py b/tests/integration_tests/modules/test_seed_random_data.py
index db3d2193..b365fa98 100644
--- a/tests/integration_tests/modules/test_seed_random_data.py
+++ b/tests/integration_tests/modules/test_seed_random_data.py
@@ -19,6 +19,7 @@ random_seed:
 """
 
 
+@pytest.mark.ci
 class TestSeedRandomData:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_set_hostname.py b/tests/integration_tests/modules/test_set_hostname.py
index ff46feb9..2bfa403d 100644
--- a/tests/integration_tests/modules/test_set_hostname.py
+++ b/tests/integration_tests/modules/test_set_hostname.py
@@ -25,6 +25,7 @@ fqdn: cloudinit2.i9n.cloud-init.io
 """
 
 
+@pytest.mark.ci
 class TestHostname:
 
     @pytest.mark.user_data(USER_DATA_HOSTNAME)
diff --git a/tests/integration_tests/modules/test_set_password.py b/tests/integration_tests/modules/test_set_password.py
index ae6fdefc..b13f76fb 100644
--- a/tests/integration_tests/modules/test_set_password.py
+++ b/tests/integration_tests/modules/test_set_password.py
@@ -139,11 +139,13 @@ class Mixin:
         assert "PasswordAuthentication yes" in sshd_config.splitlines()
 
 
+@pytest.mark.ci
 @pytest.mark.user_data(LIST_USER_DATA)
 class TestPasswordList(Mixin):
     """Launch an instance with LIST_USER_DATA, ensure Mixin tests pass."""
 
 
+@pytest.mark.ci
 @pytest.mark.user_data(STRING_USER_DATA)
 class TestPasswordListString(Mixin):
     """Launch an instance with STRING_USER_DATA, ensure Mixin tests pass."""
diff --git a/tests/integration_tests/modules/test_snap.py b/tests/integration_tests/modules/test_snap.py
index d78a0b1e..b626f6b0 100644
--- a/tests/integration_tests/modules/test_snap.py
+++ b/tests/integration_tests/modules/test_snap.py
@@ -19,6 +19,7 @@ snap:
 """
 
 
+@pytest.mark.ci
 class TestSnap:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py
index e88d9a02..b9b0d85e 100644
--- a/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py
+++ b/tests/integration_tests/modules/test_ssh_auth_key_fingerprints.py
@@ -28,6 +28,7 @@ ssh_authorized_keys:
 """  # noqa
 
 
+@pytest.mark.ci
 class TestSshAuthkeyFingerprints:
 
     @pytest.mark.user_data(USER_DATA_SSH_AUTHKEY_DISABLE)
diff --git a/tests/integration_tests/modules/test_ssh_generate.py b/tests/integration_tests/modules/test_ssh_generate.py
index 8c60fb87..60c36982 100644
--- a/tests/integration_tests/modules/test_ssh_generate.py
+++ b/tests/integration_tests/modules/test_ssh_generate.py
@@ -20,6 +20,7 @@ authkey_hash: sha512
 """
 
 
+@pytest.mark.ci
 @pytest.mark.user_data(USER_DATA)
 class TestSshKeysGenerate:
 
diff --git a/tests/integration_tests/modules/test_ssh_import_id.py b/tests/integration_tests/modules/test_ssh_import_id.py
index 2f2ac92c..45d37d6c 100644
--- a/tests/integration_tests/modules/test_ssh_import_id.py
+++ b/tests/integration_tests/modules/test_ssh_import_id.py
@@ -17,6 +17,7 @@ ssh_import_id:
 """
 
 
+@pytest.mark.ci
 class TestSshImportId:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_ssh_keys_provided.py b/tests/integration_tests/modules/test_ssh_keys_provided.py
index 4699518d..dc6d2fc1 100644
--- a/tests/integration_tests/modules/test_ssh_keys_provided.py
+++ b/tests/integration_tests/modules/test_ssh_keys_provided.py
@@ -78,6 +78,7 @@ ssh_keys:
 """  # noqa
 
 
+@pytest.mark.ci
 @pytest.mark.user_data(USER_DATA)
 class TestSshKeysProvided:
 
diff --git a/tests/integration_tests/modules/test_timezone.py b/tests/integration_tests/modules/test_timezone.py
index 6080d79e..111d53f7 100644
--- a/tests/integration_tests/modules/test_timezone.py
+++ b/tests/integration_tests/modules/test_timezone.py
@@ -15,6 +15,7 @@ timezone: US/Aleutian
 """
 
 
+@pytest.mark.ci
 class TestTimezone:
 
     @pytest.mark.user_data(USER_DATA)
diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py
index b1fa8c22..6a085a8f 100644
--- a/tests/integration_tests/modules/test_users_groups.py
+++ b/tests/integration_tests/modules/test_users_groups.py
@@ -38,6 +38,7 @@ AHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
 """
 
 
+@pytest.mark.ci
 @pytest.mark.user_data(USER_DATA)
 class TestUsersGroups:
     @pytest.mark.parametrize(
diff --git a/tests/integration_tests/modules/test_write_files.py b/tests/integration_tests/modules/test_write_files.py
index d7032a0c..15832ae3 100644
--- a/tests/integration_tests/modules/test_write_files.py
+++ b/tests/integration_tests/modules/test_write_files.py
@@ -44,6 +44,7 @@ write_files:
 """.format(B64_CONTENT.decode("ascii"))
 
 
+@pytest.mark.ci
 @pytest.mark.user_data(USER_DATA)
 class TestWriteFiles:
 
diff --git a/tox.ini b/tox.ini
index 32174dee..4320ab87 100644
--- a/tox.ini
+++ b/tox.ini
@@ -146,6 +146,13 @@ passenv = CLOUD_INIT_*
 deps =
     -r{toxinidir}/integration-requirements.txt
 
+[testenv:integration-tests-ci]
+commands = {[testenv:integration-tests]commands}
+passenv = {[testenv:integration-tests]passenv}
+deps = {[testenv:integration-tests]deps}
+setenv =
+    PYTEST_ADDOPTS="-k ci"
+
 [pytest]
 # TODO: s/--strict/--strict-markers/ once xenial support is dropped
 testpaths = cloudinit tests/unittests
@@ -153,6 +160,7 @@ addopts = --strict
 markers =
     allow_subp_for: allow subp usage for the given commands (disable_subp_usage)
     allow_all_subp: allow all subp usage (disable_subp_usage)
+    ci: run this integration test as part of CI test runs
     ds_sys_cfg: a sys_cfg dict to be used by datasource fixtures
     ec2: test will only run on EC2 platform
     gce: test will only run on GCE platform
-- 
cgit v1.2.3