Merge pull request #18 from zdc/T2117-equuleus-20.3

T2117: Cloud-init updated to 20.3

8 files changed, 213 insertions, 88 deletions

diff --git a/tests/unittests/test_distros/test_bsd_utils.py b/tests/unittests/test_distros/test_bsd_utils.py
new file mode 100644
index 00000000..3a68f2a9
--- /dev/null
+++ b/tests/unittests/test_distros/test_bsd_utils.py
@@ -0,0 +1,67 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import cloudinit.distros.bsd_utils as bsd_utils
+
+from cloudinit.tests.helpers import (CiTestCase, ExitStack, mock)
+
+RC_FILE = """
+if something; then
+    do something here
+fi
+hostname={hostname}
+"""
+
+
+class TestBsdUtils(CiTestCase):
+
+    def setUp(self):
+        super().setUp()
+        patches = ExitStack()
+        self.addCleanup(patches.close)
+
+        self.load_file = patches.enter_context(
+            mock.patch.object(bsd_utils.util, 'load_file'))
+
+        self.write_file = patches.enter_context(
+            mock.patch.object(bsd_utils.util, 'write_file'))
+
+    def test_get_rc_config_value(self):
+        self.load_file.return_value = 'hostname=foo\n'
+        self.assertEqual(bsd_utils.get_rc_config_value('hostname'), 'foo')
+        self.load_file.assert_called_with('/etc/rc.conf')
+
+        self.load_file.return_value = 'hostname=foo'
+        self.assertEqual(bsd_utils.get_rc_config_value('hostname'), 'foo')
+
+        self.load_file.return_value = 'hostname="foo"'
+        self.assertEqual(bsd_utils.get_rc_config_value('hostname'), 'foo')
+
+        self.load_file.return_value = "hostname='foo'"
+        self.assertEqual(bsd_utils.get_rc_config_value('hostname'), 'foo')
+
+        self.load_file.return_value = 'hostname=\'foo"'
+        self.assertEqual(bsd_utils.get_rc_config_value('hostname'), "'foo\"")
+
+        self.load_file.return_value = ''
+        self.assertEqual(bsd_utils.get_rc_config_value('hostname'), None)
+
+        self.load_file.return_value = RC_FILE.format(hostname='foo')
+        self.assertEqual(bsd_utils.get_rc_config_value('hostname'), "foo")
+
+    def test_set_rc_config_value_unchanged(self):
+        # bsd_utils.set_rc_config_value('hostname', 'foo')
+        # self.write_file.assert_called_with('/etc/rc.conf', 'hostname=foo\n')
+
+        self.load_file.return_value = RC_FILE.format(hostname='foo')
+        self.write_file.assert_not_called()
+
+    def test_set_rc_config_value(self):
+        bsd_utils.set_rc_config_value('hostname', 'foo')
+        self.write_file.assert_called_with('/etc/rc.conf', 'hostname=foo\n')
+
+        self.load_file.return_value = RC_FILE.format(hostname='foo')
+        bsd_utils.set_rc_config_value('hostname', 'bar')
+        self.write_file.assert_called_with(
+            '/etc/rc.conf',
+            RC_FILE.format(hostname='bar')
+        )
diff --git a/tests/unittests/test_distros/test_create_users.py b/tests/unittests/test_distros/test_create_users.py
index ef11784d..94ab052d 100644
--- a/tests/unittests/test_distros/test_create_users.py
+++ b/tests/unittests/test_distros/test_create_users.py
@@ -46,7 +46,7 @@ class MyBaseDistro(distros.Distro):
 
 
 @mock.patch("cloudinit.distros.util.system_is_snappy", return_value=False)
-@mock.patch("cloudinit.distros.util.subp")
+@mock.patch("cloudinit.distros.subp.subp")
 class TestCreateUser(CiTestCase):
 
     with_logs = True
@@ -240,7 +240,7 @@ class TestCreateUser(CiTestCase):
             [mock.call(set(['auth1']), user),  # not disabled
              mock.call(set(['key1']), 'foouser', options=disable_prefix)])
 
-    @mock.patch("cloudinit.distros.util.which")
+    @mock.patch("cloudinit.distros.subp.which")
     def test_lock_with_usermod_if_no_passwd(self, m_which, m_subp,
                                             m_is_snappy):
         """Lock uses usermod --lock if no 'passwd' cmd available."""
@@ -250,7 +250,7 @@ class TestCreateUser(CiTestCase):
             [mock.call(['usermod', '--lock', 'bob'])],
             m_subp.call_args_list)
 
-    @mock.patch("cloudinit.distros.util.which")
+    @mock.patch("cloudinit.distros.subp.which")
     def test_lock_with_passwd_if_available(self, m_which, m_subp,
                                            m_is_snappy):
         """Lock with only passwd will use passwd."""
@@ -260,7 +260,7 @@ class TestCreateUser(CiTestCase):
             [mock.call(['passwd', '-l', 'bob'])],
             m_subp.call_args_list)
 
-    @mock.patch("cloudinit.distros.util.which")
+    @mock.patch("cloudinit.distros.subp.which")
     def test_lock_raises_runtime_if_no_commands(self, m_which, m_subp,
                                                 m_is_snappy):
         """Lock with no commands available raises RuntimeError."""
diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py
index da16a797..7ff8240b 100644
--- a/tests/unittests/test_distros/test_debian.py
+++ b/tests/unittests/test_distros/test_debian.py
@@ -5,7 +5,7 @@ from cloudinit import util
 from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock)
 
 
-@mock.patch("cloudinit.distros.debian.util.subp")
+@mock.patch("cloudinit.distros.debian.subp.subp")
 class TestDebianApplyLocale(FilesystemMockingTestCase):
 
     def setUp(self):
diff --git a/tests/unittests/test_distros/test_freebsd.py b/tests/unittests/test_distros/test_freebsd.py
index 8af253a2..be565b04 100644
--- a/tests/unittests/test_distros/test_freebsd.py
+++ b/tests/unittests/test_distros/test_freebsd.py
@@ -8,7 +8,7 @@ import os
 
 class TestDeviceLookUp(CiTestCase):
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_find_freebsd_part_label(self, mock_subp):
         glabel_out = '''
 gptid/fa52d426-c337-11e6-8911-00155d4c5e47  N/A  da0p1
@@ -19,7 +19,7 @@ gptid/fa52d426-c337-11e6-8911-00155d4c5e47  N/A  da0p1
         res = find_freebsd_part("/dev/label/rootfs")
         self.assertEqual("da0p2", res)
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_find_freebsd_part_gpt(self, mock_subp):
         glabel_out = '''
                                 gpt/bootfs  N/A  vtbd0p1
diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py
index 02b334e3..44607489 100644
--- a/tests/unittests/test_distros/test_generic.py
+++ b/tests/unittests/test_distros/test_generic.py
@@ -6,6 +6,7 @@ from cloudinit import util
 from cloudinit.tests import helpers
 
 import os
+import pytest
 import shutil
 import tempfile
 from unittest import mock
@@ -37,24 +38,6 @@ gapmi = distros._get_arch_package_mirror_info
 
 class TestGenericDistro(helpers.FilesystemMockingTestCase):
 
-    def return_first(self, mlist):
-        if not mlist:
-            return None
-        return mlist[0]
-
-    def return_second(self, mlist):
-        if not mlist:
-            return None
-        return mlist[1]
-
-    def return_none(self, _mlist):
-        return None
-
-    def return_last(self, mlist):
-        if not mlist:
-            return None
-        return(mlist[-1])
-
     def setUp(self):
         super(TestGenericDistro, self).setUp()
         # Make a temp directoy for tests to use.
@@ -145,61 +128,6 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase):
         arch_mirrors = gapmi(package_mirrors, arch="amd64")
         self.assertEqual(package_mirrors[0], arch_mirrors)
 
-    def test_get_package_mirror_info_az_ec2(self):
-        arch_mirrors = gapmi(package_mirrors, arch="amd64")
-        data_source_mock = mock.Mock(availability_zone="us-east-1a")
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_first)
-        self.assertEqual(results,
-                         {'primary': 'http://us-east-1.ec2/',
-                          'security': 'http://security-mirror1-intel'})
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_second)
-        self.assertEqual(results,
-                         {'primary': 'http://us-east-1a.clouds/',
-                          'security': 'http://security-mirror2-intel'})
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_none)
-        self.assertEqual(results, package_mirrors[0]['failsafe'])
-
-    def test_get_package_mirror_info_az_non_ec2(self):
-        arch_mirrors = gapmi(package_mirrors, arch="amd64")
-        data_source_mock = mock.Mock(availability_zone="nova.cloudvendor")
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_first)
-        self.assertEqual(results,
-                         {'primary': 'http://nova.cloudvendor.clouds/',
-                          'security': 'http://security-mirror1-intel'})
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_last)
-        self.assertEqual(results,
-                         {'primary': 'http://nova.cloudvendor.clouds/',
-                          'security': 'http://security-mirror2-intel'})
-
-    def test_get_package_mirror_info_none(self):
-        arch_mirrors = gapmi(package_mirrors, arch="amd64")
-        data_source_mock = mock.Mock(availability_zone=None)
-
-        # because both search entries here replacement based on
-        # availability-zone, the filter will be called with an empty list and
-        # failsafe should be taken.
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_first)
-        self.assertEqual(results,
-                         {'primary': 'http://fs-primary-intel',
-                          'security': 'http://security-mirror1-intel'})
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_last)
-        self.assertEqual(results,
-                         {'primary': 'http://fs-primary-intel',
-                          'security': 'http://security-mirror2-intel'})
-
     def test_systemd_in_use(self):
         cls = distros.fetch("ubuntu")
         d = cls("ubuntu", {}, None)
@@ -245,7 +173,7 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase):
         for d_name in ("ubuntu", "rhel"):
             cls = distros.fetch(d_name)
             d = cls(d_name, {}, None)
-            with mock.patch("cloudinit.util.subp") as m_subp:
+            with mock.patch("cloudinit.subp.subp") as m_subp:
                 d.expire_passwd("myuser")
             m_subp.assert_called_once_with(["passwd", "--expire", "myuser"])
 
@@ -253,10 +181,122 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase):
         """Test FreeBSD.expire_passwd uses the pw command."""
         cls = distros.fetch("freebsd")
         d = cls("freebsd", {}, None)
-        with mock.patch("cloudinit.util.subp") as m_subp:
+        with mock.patch("cloudinit.subp.subp") as m_subp:
             d.expire_passwd("myuser")
         m_subp.assert_called_once_with(
             ["pw", "usermod", "myuser", "-p", "01-Jan-1970"])
 
 
+class TestGetPackageMirrors:
+
+    def return_first(self, mlist):
+        if not mlist:
+            return None
+        return mlist[0]
+
+    def return_second(self, mlist):
+        if not mlist:
+            return None
+
+        return mlist[1] if len(mlist) > 1 else None
+
+    def return_none(self, _mlist):
+        return None
+
+    def return_last(self, mlist):
+        if not mlist:
+            return None
+        return(mlist[-1])
+
+    @pytest.mark.parametrize(
+        "allow_ec2_mirror, platform_type, mirrors",
+        [
+            (True, "ec2", [
+                {'primary': 'http://us-east-1.ec2/',
+                 'security': 'http://security-mirror1-intel'},
+                {'primary': 'http://us-east-1a.clouds/',
+                 'security': 'http://security-mirror2-intel'}
+            ]),
+            (True, "other", [
+                {'primary': 'http://us-east-1.ec2/',
+                 'security': 'http://security-mirror1-intel'},
+                {'primary': 'http://us-east-1a.clouds/',
+                 'security': 'http://security-mirror2-intel'}
+            ]),
+            (False, "ec2", [
+                {'primary': 'http://us-east-1.ec2/',
+                 'security': 'http://security-mirror1-intel'},
+                {'primary': 'http://us-east-1a.clouds/',
+                 'security': 'http://security-mirror2-intel'}
+            ]),
+            (False, "other", [
+                {'primary': 'http://us-east-1a.clouds/',
+                 'security': 'http://security-mirror1-intel'},
+                {'primary': 'http://fs-primary-intel',
+                 'security': 'http://security-mirror2-intel'}
+            ])
+        ])
+    def test_get_package_mirror_info_az_ec2(self,
+                                            allow_ec2_mirror,
+                                            platform_type,
+                                            mirrors):
+        flag_path = "cloudinit.distros." \
+                    "ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES"
+        with mock.patch(flag_path, allow_ec2_mirror):
+            arch_mirrors = gapmi(package_mirrors, arch="amd64")
+            data_source_mock = mock.Mock(
+                availability_zone="us-east-1a",
+                platform_type=platform_type)
+
+            results = gpmi(arch_mirrors, data_source=data_source_mock,
+                           mirror_filter=self.return_first)
+            assert(results == mirrors[0])
+
+            results = gpmi(arch_mirrors, data_source=data_source_mock,
+                           mirror_filter=self.return_second)
+            assert(results == mirrors[1])
+
+            results = gpmi(arch_mirrors, data_source=data_source_mock,
+                           mirror_filter=self.return_none)
+            assert(results == package_mirrors[0]['failsafe'])
+
+    def test_get_package_mirror_info_az_non_ec2(self):
+        arch_mirrors = gapmi(package_mirrors, arch="amd64")
+        data_source_mock = mock.Mock(availability_zone="nova.cloudvendor")
+
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
+                       mirror_filter=self.return_first)
+        assert(results == {
+            'primary': 'http://nova.cloudvendor.clouds/',
+            'security': 'http://security-mirror1-intel'}
+        )
+
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
+                       mirror_filter=self.return_last)
+        assert(results == {
+            'primary': 'http://nova.cloudvendor.clouds/',
+            'security': 'http://security-mirror2-intel'}
+        )
+
+    def test_get_package_mirror_info_none(self):
+        arch_mirrors = gapmi(package_mirrors, arch="amd64")
+        data_source_mock = mock.Mock(availability_zone=None)
+
+        # because both search entries here replacement based on
+        # availability-zone, the filter will be called with an empty list and
+        # failsafe should be taken.
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
+                       mirror_filter=self.return_first)
+        assert(results == {
+            'primary': 'http://fs-primary-intel',
+            'security': 'http://security-mirror1-intel'}
+        )
+
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
+                       mirror_filter=self.return_last)
+        assert(results == {
+            'primary': 'http://fs-primary-intel',
+            'security': 'http://security-mirror2-intel'}
+        )
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_distros/test_netbsd.py b/tests/unittests/test_distros/test_netbsd.py
new file mode 100644
index 00000000..11a68d2a
--- /dev/null
+++ b/tests/unittests/test_distros/test_netbsd.py
@@ -0,0 +1,17 @@
+import cloudinit.distros.netbsd
+
+import pytest
+import unittest.mock as mock
+
+
+@pytest.mark.parametrize('with_pkgin', (True, False))
+@mock.patch("cloudinit.distros.netbsd.os")
+def test_init(m_os, with_pkgin):
+    print(with_pkgin)
+    m_os.path.exists.return_value = with_pkgin
+    cfg = {}
+
+    distro = cloudinit.distros.netbsd.NetBSD("netbsd", cfg, None)
+    expectation = ['pkgin', '-y', 'full-upgrade'] if with_pkgin else None
+    assert distro.pkg_cmd_upgrade_prefix == expectation
+    assert [mock.call('/usr/pkg/bin/pkgin')] == m_os.path.exists.call_args_list
diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py
index ccf66161..8d7b09c8 100644
--- a/tests/unittests/test_distros/test_netconfig.py
+++ b/tests/unittests/test_distros/test_netconfig.py
@@ -12,6 +12,7 @@ from cloudinit import helpers
 from cloudinit import settings
 from cloudinit.tests.helpers import (
     FilesystemMockingTestCase, dir2dict)
+from cloudinit import subp
 from cloudinit import util
 
 
@@ -532,7 +533,7 @@ class TestNetCfgDistroRedhat(TestNetCfgDistroBase):
                 NETWORKING_IPV6=yes
                 IPV6_AUTOCONF=no
                 """),
-            }
+        }
         # rh_distro.apply_network_config(V1_NET_CFG_IPV6, False)
         self._apply_and_verify(self.distro.apply_network_config,
                                V1_NET_CFG_IPV6,
@@ -656,7 +657,7 @@ class TestNetCfgDistroArch(TestNetCfgDistroBase):
                 IP=dhcp
                 Interface=eth1
                 """),
-            }
+        }
 
         # ub_distro.apply_network_config(V1_NET_CFG, False)
         self._apply_and_verify(self.distro.apply_network_config,
@@ -688,6 +689,6 @@ class TestNetCfgDistroArch(TestNetCfgDistroBase):
 
 
 def get_mode(path, target=None):
-    return os.stat(util.target_path(target, path)).st_mode & 0o777
+    return os.stat(subp.target_path(target, path)).st_mode & 0o777
 
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py
index a6faf0ef..fa48410a 100644
--- a/tests/unittests/test_distros/test_user_data_normalize.py
+++ b/tests/unittests/test_distros/test_user_data_normalize.py
@@ -307,7 +307,7 @@ class TestUGNormalize(TestCase):
         self.assertEqual({'default': False}, users['joe'])
         self.assertEqual({'default': False}, users['bob'])
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_create_snap_user(self, mock_subp):
         mock_subp.side_effect = [('{"username": "joe", "ssh-key-count": 1}\n',
                                   '')]
@@ -326,7 +326,7 @@ class TestUGNormalize(TestCase):
         mock_subp.assert_called_with(snapcmd, capture=True, logstring=snapcmd)
         self.assertEqual(username, 'joe')
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_create_snap_user_known(self, mock_subp):
         mock_subp.side_effect = [('{"username": "joe", "ssh-key-count": 1}\n',
                                   '')]
@@ -348,7 +348,7 @@ class TestUGNormalize(TestCase):
 
     @mock.patch('cloudinit.util.system_is_snappy')
     @mock.patch('cloudinit.util.is_group')
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_add_user_on_snappy_system(self, mock_subp, mock_isgrp,
                                        mock_snappy):
         mock_isgrp.return_value = False