FreeBSD: improvements and fixes for use on Azure

This patch targets to make FreeBSD 10.3 or 11 work on Azure. The
modifications abide by the rule of:
  * making as less modification as possible
  * delegate to the distro or datasource where possible.

The main modifications are:
 1. network configuration improvements, and movement into distro path.
 2. Fix setting of password.
    Password setting through "pw" can only work through pipe.
 3. Add 'root:wheel' to syslog_fix_perms field.
 4. Support resizing default file system (ufs)
 5. copy cloud.cfg for freebsd to /etc/cloud/cloud.cfg rather than
    /usr/local/etc/cloud/cloud.cfg.
 6. Azure specific changes:
    a. When reading the azure endpoint, search in a different path
       and read a different option name (option-245 vs. unknown-245).
        so, the lease file path should be generated according to platform.
    b. adjust the handling of ephemeral mounts for ufs filesystem and
       for finding the ephemeral device.
    c. fix mounting of cdrom

LP: #1636345

7 files changed, 182 insertions, 7 deletions

diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 8d22bb59..e6b0dcb4 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -3,6 +3,8 @@
 from cloudinit import helpers
 from cloudinit.util import b64e, decode_binary, load_file
 from cloudinit.sources import DataSourceAzure
+from cloudinit.util import find_freebsd_part
+from cloudinit.util import get_path_dev_freebsd
 
 from ..helpers import TestCase, populate_dir, mock, ExitStack, PY26, SkipTest
 
@@ -95,6 +97,41 @@ class TestAzureDataSource(TestCase):
         for module, name, new in patches:
             self.patches.enter_context(mock.patch.object(module, name, new))
 
+    def _get_mockds(self):
+        mod = DataSourceAzure
+        sysctl_out = "dev.storvsc.3.%pnpinfo: "\
+                     "classid=ba6163d9-04a1-4d29-b605-72e2ffb1dc7f "\
+                     "deviceid=f8b3781b-1e82-4818-a1c3-63d806ec15bb\n"
+        sysctl_out += "dev.storvsc.2.%pnpinfo: "\
+                      "classid=ba6163d9-04a1-4d29-b605-72e2ffb1dc7f "\
+                      "deviceid=f8b3781a-1e82-4818-a1c3-63d806ec15bb\n"
+        sysctl_out += "dev.storvsc.1.%pnpinfo: "\
+                      "classid=32412632-86cb-44a2-9b5c-50d1417354f5 "\
+                      "deviceid=00000000-0001-8899-0000-000000000000\n"
+        camctl_devbus = """
+scbus0 on ata0 bus 0
+scbus1 on ata1 bus 0
+scbus2 on blkvsc0 bus 0
+scbus3 on blkvsc1 bus 0
+scbus4 on storvsc2 bus 0
+scbus5 on storvsc3 bus 0
+scbus-1 on xpt0 bus 0
+        """
+        camctl_dev = """
+<Msft Virtual CD/ROM 1.0>          at scbus1 target 0 lun 0 (cd0,pass0)
+<Msft Virtual Disk 1.0>            at scbus2 target 0 lun 0 (da0,pass1)
+<Msft Virtual Disk 1.0>            at scbus3 target 1 lun 0 (da1,pass2)
+        """
+        self.apply_patches([
+            (mod, 'get_dev_storvsc_sysctl', mock.MagicMock(
+                return_value=sysctl_out)),
+            (mod, 'get_camcontrol_dev_bus', mock.MagicMock(
+                return_value=camctl_devbus)),
+            (mod, 'get_camcontrol_dev', mock.MagicMock(
+                return_value=camctl_dev))
+        ])
+        return mod
+
     def _get_ds(self, data, agent_command=None):
 
         def dsdevs():
@@ -177,6 +214,34 @@ class TestAzureDataSource(TestCase):
             return
         raise AssertionError("XML is the same")
 
+    def test_get_resource_disk(self):
+        ds = self._get_mockds()
+        dev = ds.get_resource_disk_on_freebsd(1)
+        self.assertEqual("da1", dev)
+
+    @mock.patch('cloudinit.util.subp')
+    def test_find_freebsd_part_on_Azure(self, mock_subp):
+        glabel_out = '''
+gptid/fa52d426-c337-11e6-8911-00155d4c5e47  N/A  da0p1
+                              label/rootfs  N/A  da0p2
+                                label/swap  N/A  da0p3
+'''
+        mock_subp.return_value = (glabel_out, "")
+        res = find_freebsd_part("/dev/label/rootfs")
+        self.assertEqual("da0p2", res)
+
+    def test_get_path_dev_freebsd_on_Azure(self):
+        mnt_list = '''
+/dev/label/rootfs  /                ufs     rw              1 1
+devfs              /dev             devfs   rw,multilabel   0 0
+fdescfs            /dev/fd          fdescfs rw              0 0
+/dev/da1s1         /mnt/resource    ufs     rw              2 2
+'''
+        with mock.patch.object(os.path, 'exists',
+                               return_value=True):
+            res = get_path_dev_freebsd('/etc', mnt_list)
+            self.assertNotEqual(res, None)
+
     def test_basic_seed_dir(self):
         odata = {'HostName': "myhost", 'UserName': "myuser"}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata),
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index aafdebd7..b2d2971b 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -3,7 +3,6 @@
 import os
 
 from cloudinit.sources.helpers import azure as azure_helper
-
 from ..helpers import ExitStack, mock, TestCase
 
 
@@ -72,10 +71,11 @@ class TestFindEndpoint(TestCase):
 
     @staticmethod
     def _build_lease_content(encoded_address):
+        endpoint = azure_helper._get_dhcp_endpoint_option_name()
         return '\n'.join([
             'lease {',
             ' interface "eth0";',
-            ' option unknown-245 {0};'.format(encoded_address),
+            ' option {0} {1};'.format(endpoint, encoded_address),
             '}'])
 
     def test_from_dhcp_client(self):
diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py
index e93d28de..1d3d2f19 100644
--- a/tests/unittests/test_datasource/test_cloudstack.py
+++ b/tests/unittests/test_datasource/test_cloudstack.py
@@ -15,6 +15,11 @@ class TestCloudStackPasswordFetching(TestCase):
         mod_name = 'cloudinit.sources.DataSourceCloudStack'
         self.patches.enter_context(mock.patch('{0}.ec2'.format(mod_name)))
         self.patches.enter_context(mock.patch('{0}.uhelp'.format(mod_name)))
+        default_gw = "192.201.20.0"
+        mod_name = 'cloudinit.sources.DataSourceCloudStack.get_default_gateway'
+        get_default_gw = mock.MagicMock(return_value=default_gw)
+        self.patches.enter_context(
+            mock.patch(mod_name, get_default_gw))
 
     def _set_password_server_response(self, response_string):
         subp = mock.MagicMock(return_value=(response_string, ''))
diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py
index 88370669..1e10a33d 100644
--- a/tests/unittests/test_distros/test_netconfig.py
+++ b/tests/unittests/test_distros/test_netconfig.py
@@ -178,6 +178,20 @@ class WriteBuffer(object):
 
 class TestNetCfgDistro(TestCase):
 
+    frbsd_ifout = """\
+hn0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
+        options=51b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,TSO4,LRO>
+        ether 00:15:5d:4c:73:00
+        inet6 fe80::215:5dff:fe4c:7300%hn0 prefixlen 64 scopeid 0x2
+        inet 10.156.76.127 netmask 0xfffffc00 broadcast 10.156.79.255
+        nd6 options=23<PERFORMNUD,ACCEPT_RTADV,AUTO_LINKLOCAL>
+        media: Ethernet autoselect (10Gbase-T <full-duplex>)
+        status: active
+"""
+
+    def setUp(self):
+        super(TestNetCfgDistro, self).setUp()
+
     def _get_distro(self, dname, renderers=None):
         cls = distros.fetch(dname)
         cfg = settings.CFG_BUILTIN
@@ -251,6 +265,7 @@ class TestNetCfgDistro(TestCase):
 
     def test_apply_network_config_v1_to_netplan_ub(self):
         renderers = ['netplan']
+        devlist = ['eth0', 'lo']
         ub_distro = self._get_distro('ubuntu', renderers=renderers)
         with ExitStack() as mocks:
             write_bufs = {}
@@ -272,6 +287,9 @@ class TestNetCfgDistro(TestCase):
                 mock.patch.object(util, 'subp', return_value=(0, 0)))
             mocks.enter_context(
                 mock.patch.object(os.path, 'isfile', return_value=False))
+            mocks.enter_context(
+                mock.patch("cloudinit.net.netplan.get_devicelist",
+                           return_value=devlist))
 
             ub_distro.apply_network_config(V1_NET_CFG, False)
 
@@ -285,6 +303,7 @@ class TestNetCfgDistro(TestCase):
 
     def test_apply_network_config_v2_passthrough_ub(self):
         renderers = ['netplan']
+        devlist = ['eth0', 'lo']
         ub_distro = self._get_distro('ubuntu', renderers=renderers)
         with ExitStack() as mocks:
             write_bufs = {}
@@ -306,7 +325,10 @@ class TestNetCfgDistro(TestCase):
                 mock.patch.object(util, 'subp', return_value=(0, 0)))
             mocks.enter_context(
                 mock.patch.object(os.path, 'isfile', return_value=False))
-
+            # FreeBSD does not have '/sys/class/net' file,
+            # so we need mock here.
+            mocks.enter_context(
+                mock.patch.object(os, 'listdir', return_value=devlist))
             ub_distro.apply_network_config(V2_NET_CFG, False)
 
             self.assertEqual(len(write_bufs), 1)
@@ -328,6 +350,29 @@ class TestNetCfgDistro(TestCase):
         for (k, v) in b1.items():
             self.assertEqual(v, b2[k])
 
+    @mock.patch('cloudinit.distros.freebsd.Distro.get_ifconfig_list')
+    @mock.patch('cloudinit.distros.freebsd.Distro.get_ifconfig_ifname_out')
+    def test_get_ip_nic_freebsd(self, ifname_out, iflist):
+        frbsd_distro = self._get_distro('freebsd')
+        iflist.return_value = "lo0 hn0"
+        ifname_out.return_value = self.frbsd_ifout
+        res = frbsd_distro.get_ipv4()
+        self.assertEqual(res, ['lo0', 'hn0'])
+        res = frbsd_distro.get_ipv6()
+        self.assertEqual(res, [])
+
+    @mock.patch('cloudinit.distros.freebsd.Distro.get_ifconfig_ether')
+    @mock.patch('cloudinit.distros.freebsd.Distro.get_ifconfig_ifname_out')
+    @mock.patch('cloudinit.distros.freebsd.Distro.get_interface_mac')
+    def test_generate_fallback_config_freebsd(self, mac, ifname_out, if_ether):
+        frbsd_distro = self._get_distro('freebsd')
+
+        if_ether.return_value = 'hn0'
+        ifname_out.return_value = self.frbsd_ifout
+        mac.return_value = '00:15:5d:4c:73:00'
+        res = frbsd_distro.generate_fallback_config()
+        self.assertIsNotNone(res)
+
     def test_simple_write_rh(self):
         rh_distro = self._get_distro('rhel')
 
diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py
new file mode 100644
index 00000000..52591b8b
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_resizefs.py
@@ -0,0 +1,59 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit.config import cc_resizefs
+
+import textwrap
+import unittest
+
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
+
+class TestResizefs(unittest.TestCase):
+    def setUp(self):
+        super(TestResizefs, self).setUp()
+        self.name = "resizefs"
+
+    @mock.patch('cloudinit.config.cc_resizefs._get_dumpfs_output')
+    @mock.patch('cloudinit.config.cc_resizefs._get_gpart_output')
+    def test_skip_ufs_resize(self, gpart_out, dumpfs_out):
+        fs_type = "ufs"
+        resize_what = "/"
+        devpth = "/dev/da0p2"
+        dumpfs_out.return_value = (
+            "# newfs command for / (/dev/label/rootfs)\n"
+            "newfs -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 "
+            "-f 4096 -g 16384 -h 64 -i 8192 -j -k 6408 -m 8 "
+            "-o time -s 58719232 /dev/label/rootfs\n")
+        gpart_out.return_value = textwrap.dedent("""\
+            =>      40  62914480  da0  GPT  (30G)
+                    40      1024    1  freebsd-boot  (512K)
+                  1064  58719232    2  freebsd-ufs  (28G)
+              58720296   3145728    3  freebsd-swap  (1.5G)
+              61866024   1048496       - free -  (512M)
+            """)
+        res = cc_resizefs.can_skip_resize(fs_type, resize_what, devpth)
+        self.assertTrue(res)
+
+    @mock.patch('cloudinit.config.cc_resizefs._get_dumpfs_output')
+    @mock.patch('cloudinit.config.cc_resizefs._get_gpart_output')
+    def test_skip_ufs_resize_roundup(self, gpart_out, dumpfs_out):
+        fs_type = "ufs"
+        resize_what = "/"
+        devpth = "/dev/da0p2"
+        dumpfs_out.return_value = (
+            "# newfs command for / (/dev/label/rootfs)\n"
+            "newfs -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 "
+            "-f 4096 -g 16384 -h 64 -i 8192 -j -k 368 -m 8 "
+            "-o time -s 297080 /dev/label/rootfs\n")
+        gpart_out.return_value = textwrap.dedent("""\
+            =>      34  297086  da0  GPT  (145M)
+                    34  297086    1  freebsd-ufs  (145M)
+            """)
+        res = cc_resizefs.can_skip_resize(fs_type, resize_what, devpth)
+        self.assertTrue(res)
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index 89e75369..5d2dd031 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -1120,14 +1120,14 @@ class TestNetplanPostcommands(CiTestCase):
         render_target = 'netplan.yaml'
         renderer = netplan.Renderer(
             {'netplan_path': render_target, 'postcmds': True})
-        renderer.render_network_state(render_dir, ns)
-
         expected = [
             mock.call(['netplan', 'generate'], capture=True),
             mock.call(['udevadm', 'test-builtin', 'net_setup_link',
                        '/sys/class/net/lo'], capture=True),
         ]
-        mock_subp.assert_has_calls(expected)
+        with mock.patch.object(os.path, 'islink', return_value=True):
+            renderer.render_network_state(render_dir, ns)
+            mock_subp.assert_has_calls(expected)
 
 
 class TestEniNetworkStateToEni(CiTestCase):
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 5d21b4b7..189caca8 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -596,7 +596,8 @@ class TestSubp(helpers.TestCase):
     def test_subp_capture_stderr(self):
         data = b'hello world'
         (out, err) = util.subp(self.stdin2err, capture=True,
-                               decode=False, data=data)
+                               decode=False, data=data,
+                               update_env={'LC_ALL': 'C'})
         self.assertEqual(err, data)
         self.assertEqual(out, b'')