integration_tests: implement citest tests run in Travis (#605)

Specifically:
* `apt_configure_sources_list`
* `ntp_servers`
* `set_password_list`
* `users_groups`

Although not currently run in Travis, `set_password_list_string` was
ported over alongside `set_password_list` (as `test_set_password`).

4 files changed, 335 insertions, 0 deletions

diff --git a/tests/integration_tests/modules/test_apt_configure_sources_list.py b/tests/integration_tests/modules/test_apt_configure_sources_list.py
new file mode 100644
index 00000000..d64b3956
--- /dev/null
+++ b/tests/integration_tests/modules/test_apt_configure_sources_list.py
@@ -0,0 +1,50 @@
+"""Integration test for the apt module's ``sources_list`` functionality.
+
+This test specifies a ``sources_list`` and then checks that (a) the expected
+number of sources.list entries is present, and (b) that each expected line
+appears in the file.
+
+(This is ported from
+``tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml``.)"""
+import re
+
+import pytest
+
+
+USER_DATA = """\
+#cloud-config
+apt:
+  primary:
+    - arches: [default]
+      uri: http://archive.ubuntu.com/ubuntu
+  security:
+    - arches: [default]
+      uri: http://security.ubuntu.com/ubuntu
+  sources_list: |
+    deb $MIRROR $RELEASE main restricted
+    deb-src $MIRROR $RELEASE main restricted
+    deb $PRIMARY $RELEASE universe restricted
+    deb-src $PRIMARY $RELEASE universe restricted
+    deb $SECURITY $RELEASE-security multiverse
+    deb-src $SECURITY $RELEASE-security multiverse
+"""
+
+EXPECTED_REGEXES = [
+    r"deb http://archive.ubuntu.com/ubuntu [a-z].* main restricted",
+    r"deb-src http://archive.ubuntu.com/ubuntu [a-z].* main restricted",
+    r"deb http://archive.ubuntu.com/ubuntu [a-z].* universe restricted",
+    r"deb-src http://archive.ubuntu.com/ubuntu [a-z].* universe restricted",
+    r"deb http://security.ubuntu.com/ubuntu [a-z].*security multiverse",
+    r"deb-src http://security.ubuntu.com/ubuntu [a-z].*security multiverse",
+]
+
+
+class TestAptConfigureSourcesList:
+
+    @pytest.mark.user_data(USER_DATA)
+    def test_sources_list(self, client):
+        sources_list = client.read_from_file("/etc/apt/sources.list")
+        assert 6 == len(sources_list.rstrip().split('\n'))
+
+        for expected_re in EXPECTED_REGEXES:
+            assert re.search(expected_re, sources_list) is not None
diff --git a/tests/integration_tests/modules/test_ntp_servers.py b/tests/integration_tests/modules/test_ntp_servers.py
new file mode 100644
index 00000000..4cad8926
--- /dev/null
+++ b/tests/integration_tests/modules/test_ntp_servers.py
@@ -0,0 +1,57 @@
+"""Integration test for the ntp module's ``servers`` functionality with ntp.
+
+This test specifies the use of the `ntp` NTP client, and ensures that the given
+NTP servers are configured as expected.
+
+(This is ported from ``tests/cloud_tests/testcases/modules/ntp_servers.yaml``.)
+"""
+import re
+
+import yaml
+import pytest
+
+USER_DATA = """\
+#cloud-config
+ntp:
+  ntp_client: ntp
+  servers:
+      - 172.16.15.14
+      - 172.16.17.18
+"""
+
+EXPECTED_SERVERS = yaml.safe_load(USER_DATA)["ntp"]["servers"]
+
+
+@pytest.mark.user_data(USER_DATA)
+class TestNtpServers:
+
+    def test_ntp_installed(self, class_client):
+        """Test that `ntpd --version` succeeds, indicating installation."""
+        result = class_client.execute("ntpd --version")
+        assert 0 == result.return_code
+
+    def test_dist_config_file_is_empty(self, class_client):
+        """Test that the distributed config file is empty.
+
+        (This test is skipped on all currently supported Ubuntu releases, so
+        may not actually be needed any longer.)
+        """
+        if class_client.execute("test -e /etc/ntp.conf.dist").failed:
+            pytest.skip("/etc/ntp.conf.dist does not exist")
+        dist_file = class_client.read_from_file("/etc/ntp.conf.dist")
+        assert 0 == len(dist_file.strip().splitlines())
+
+    def test_ntp_entries(self, class_client):
+        ntp_conf = class_client.read_from_file("/etc/ntp.conf")
+        for expected_server in EXPECTED_SERVERS:
+            assert re.search(
+                r"^server {} iburst".format(expected_server),
+                ntp_conf,
+                re.MULTILINE
+            )
+
+    def test_ntpq_servers(self, class_client):
+        result = class_client.execute("ntpq -p -w -n")
+        assert result.ok
+        for expected_server in EXPECTED_SERVERS:
+            assert expected_server in result.stdout
diff --git a/tests/integration_tests/modules/test_set_password.py b/tests/integration_tests/modules/test_set_password.py
new file mode 100644
index 00000000..ae6fdefc
--- /dev/null
+++ b/tests/integration_tests/modules/test_set_password.py
@@ -0,0 +1,149 @@
+"""Integration test for the set_password module.
+
+This test specifies a combination of user/password pairs, and ensures that the
+system has the correct passwords set.
+
+There are two tests run here: one tests chpasswd's list being a YAML list, the
+other tests chpasswd's list being a string.  Both expect the same results, so
+they use a mixin to share their test definitions, because we can (of course)
+only specify one user-data per instance.
+"""
+import crypt
+
+import pytest
+import yaml
+
+
+COMMON_USER_DATA = """\
+#cloud-config
+ssh_pwauth: yes
+users:
+  - default
+  - name: tom
+    # md5 gotomgo
+    passwd: "$1$S7$tT1BEDIYrczeryDQJfdPe0"
+    lock_passwd: false
+  - name: dick
+    # md5 gocubsgo
+    passwd: "$1$ssisyfpf$YqvuJLfrrW6Cg/l53Pi1n1"
+    lock_passwd: false
+  - name: harry
+    # sha512 goharrygo
+    passwd: "$6$LF$9Z2p6rWK6TNC1DC6393ec0As.18KRAvKDbfsGJEdWN3sRQRwpdfoh37EQ3y\
+Uh69tP4GSrGW5XKHxMLiKowJgm/"
+    lock_passwd: false
+  - name: jane
+    # sha256 gojanego
+    passwd: "$5$iW$XsxmWCdpwIW8Yhv.Jn/R3uk6A4UaicfW5Xp7C9p9pg."
+    lock_passwd: false
+  - name: "mikey"
+    lock_passwd: false
+"""
+
+LIST_USER_DATA = COMMON_USER_DATA + """
+chpasswd:
+  list:
+    - tom:mypassword123!
+    - dick:RANDOM
+    - harry:RANDOM
+    - mikey:$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89
+"""
+
+STRING_USER_DATA = COMMON_USER_DATA + """
+chpasswd:
+    list: |
+      tom:mypassword123!
+      dick:RANDOM
+      harry:RANDOM
+      mikey:$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89
+"""
+
+USERS_DICTS = yaml.safe_load(COMMON_USER_DATA)["users"]
+USERS_PASSWD_VALUES = {
+    user_dict["name"]: user_dict["passwd"]
+    for user_dict in USERS_DICTS
+    if "name" in user_dict and "passwd" in user_dict
+}
+
+
+class Mixin:
+    """Shared test definitions."""
+
+    def _fetch_and_parse_etc_shadow(self, class_client):
+        """Fetch /etc/shadow and parse it into Python data structures
+
+        Returns: ({user: password}, [duplicate, users])
+        """
+        shadow_content = class_client.read_from_file("/etc/shadow")
+        users = {}
+        dupes = []
+        for line in shadow_content.splitlines():
+            user, encpw = line.split(":")[0:2]
+            if user in users:
+                dupes.append(user)
+            users[user] = encpw
+        return users, dupes
+
+    def test_no_duplicate_users_in_shadow(self, class_client):
+        """Confirm that set_passwords has not added duplicate shadow entries"""
+        _, dupes = self._fetch_and_parse_etc_shadow(class_client)
+
+        assert [] == dupes
+
+    def test_password_in_users_dict_set_correctly(self, class_client):
+        """Test that the password specified in the users dict is set."""
+        shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client)
+        assert USERS_PASSWD_VALUES["jane"] == shadow_users["jane"]
+
+    def test_password_in_chpasswd_list_set_correctly(self, class_client):
+        """Test that a chpasswd password overrides one in the users dict."""
+        shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client)
+        mikey_hash = "$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89"
+        assert mikey_hash == shadow_users["mikey"]
+
+    def test_random_passwords_set_correctly(self, class_client):
+        """Test that RANDOM chpasswd entries replace users dict passwords."""
+        shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client)
+
+        # These should have been changed
+        assert shadow_users["harry"] != USERS_PASSWD_VALUES["harry"]
+        assert shadow_users["dick"] != USERS_PASSWD_VALUES["dick"]
+
+        # To random passwords
+        assert shadow_users["harry"].startswith("$")
+        assert shadow_users["dick"].startswith("$")
+
+        # Which are not the same
+        assert shadow_users["harry"] != shadow_users["dick"]
+
+    def test_explicit_password_set_correctly(self, class_client):
+        """Test that an explicitly-specified password is set correctly."""
+        shadow_users, _ = self._fetch_and_parse_etc_shadow(class_client)
+
+        fmt_and_salt = shadow_users["tom"].rsplit("$", 1)[0]
+        expected_value = crypt.crypt("mypassword123!", fmt_and_salt)
+
+        assert expected_value == shadow_users["tom"]
+
+    def test_shadow_expected_users(self, class_client):
+        """Test that the right set of users is in /etc/shadow."""
+        shadow = class_client.read_from_file("/etc/shadow")
+        for user_dict in USERS_DICTS:
+            if "name" in user_dict:
+                assert "{}:".format(user_dict["name"]) in shadow
+
+    def test_sshd_config(self, class_client):
+        """Test that SSH password auth is enabled."""
+        sshd_config = class_client.read_from_file("/etc/ssh/sshd_config")
+        # We look for the exact line match, to avoid a commented line matching
+        assert "PasswordAuthentication yes" in sshd_config.splitlines()
+
+
+@pytest.mark.user_data(LIST_USER_DATA)
+class TestPasswordList(Mixin):
+    """Launch an instance with LIST_USER_DATA, ensure Mixin tests pass."""
+
+
+@pytest.mark.user_data(STRING_USER_DATA)
+class TestPasswordListString(Mixin):
+    """Launch an instance with STRING_USER_DATA, ensure Mixin tests pass."""
diff --git a/tests/integration_tests/modules/test_users_groups.py b/tests/integration_tests/modules/test_users_groups.py
new file mode 100644
index 00000000..b1fa8c22
--- /dev/null
+++ b/tests/integration_tests/modules/test_users_groups.py
@@ -0,0 +1,79 @@
+"""Integration test for the user_groups module.
+
+This test specifies a number of users and groups via user-data, and confirms
+that they have been configured correctly in the system under test.
+"""
+import re
+
+import pytest
+
+
+USER_DATA = """\
+#cloud-config
+# Add groups to the system
+groups:
+  - secret: [root]
+  - cloud-users
+
+# Add users to the system. Users are added after groups are added.
+users:
+  - default
+  - name: foobar
+    gecos: Foo B. Bar
+    primary_group: foobar
+    groups: users
+    expiredate: 2038-01-19
+    lock_passwd: false
+    passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYe\
+AHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
+  - name: barfoo
+    gecos: Bar B. Foo
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    groups: [cloud-users, secret]
+    lock_passwd: true
+  - name: cloudy
+    gecos: Magic Cloud App Daemon User
+    inactive: true
+    system: true
+"""
+
+
+@pytest.mark.user_data(USER_DATA)
+class TestUsersGroups:
+    @pytest.mark.parametrize(
+        "getent_args,regex",
+        [
+            # Test the ubuntu group
+            (["group", "ubuntu"], r"ubuntu:x:[0-9]{4}:"),
+            # Test the cloud-users group
+            (["group", "cloud-users"], r"cloud-users:x:[0-9]{4}:barfoo"),
+            # Test the ubuntu user
+            (
+                ["passwd", "ubuntu"],
+                r"ubuntu:x:[0-9]{4}:[0-9]{4}:Ubuntu:/home/ubuntu:/bin/bash",
+            ),
+            # Test the foobar user
+            (
+                ["passwd", "foobar"],
+                r"foobar:x:[0-9]{4}:[0-9]{4}:Foo B. Bar:/home/foobar:",
+            ),
+            # Test the barfoo user
+            (
+                ["passwd", "barfoo"],
+                r"barfoo:x:[0-9]{4}:[0-9]{4}:Bar B. Foo:/home/barfoo:",
+            ),
+            # Test the cloudy user
+            (["passwd", "cloudy"], r"cloudy:x:[0-9]{3,4}:"),
+        ],
+    )
+    def test_users_groups(self, regex, getent_args, class_client):
+        """Use getent to interrogate the various expected outcomes"""
+        result = class_client.execute(["getent"] + getent_args)
+        assert re.search(regex, result.stdout) is not None
+
+    def test_user_root_in_secret(self, class_client):
+        """Test root user is in 'secret' group."""
+        output = class_client.execute("groups root").stdout
+        _, groups_str = output.split(":", maxsplit=1)
+        groups = groups_str.split()
+        assert "secret" in groups