FreeBSD: improvements and fixes for use on Azure

This patch targets to make FreeBSD 10.3 or 11 work on Azure. The
modifications abide by the rule of:
  * making as less modification as possible
  * delegate to the distro or datasource where possible.

The main modifications are:
 1. network configuration improvements, and movement into distro path.
 2. Fix setting of password.
    Password setting through "pw" can only work through pipe.
 3. Add 'root:wheel' to syslog_fix_perms field.
 4. Support resizing default file system (ufs)
 5. copy cloud.cfg for freebsd to /etc/cloud/cloud.cfg rather than
    /usr/local/etc/cloud/cloud.cfg.
 6. Azure specific changes:
    a. When reading the azure endpoint, search in a different path
       and read a different option name (option-245 vs. unknown-245).
        so, the lease file path should be generated according to platform.
    b. adjust the handling of ephemeral mounts for ufs filesystem and
       for finding the ephemeral device.
    c. fix mounting of cdrom

LP: #1636345

2 files changed, 179 insertions, 12 deletions

diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 04358b73..5254e18a 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -10,6 +10,7 @@ import crypt
 from functools import partial
 import os
 import os.path
+import re
 import time
 from xml.dom import minidom
 import xml.etree.ElementTree as ET
@@ -32,19 +33,160 @@ BOUNCE_COMMAND = [
 # azure systems will always have a resource disk, and 66-azure-ephemeral.rules
 # ensures that it gets linked to this path.
 RESOURCE_DISK_PATH = '/dev/disk/cloud/azure_resource'
+DEFAULT_PRIMARY_NIC = 'eth0'
+LEASE_FILE = '/var/lib/dhcp/dhclient.eth0.leases'
+DEFAULT_FS = 'ext4'
+
+
+def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid):
+    # extract the 'X' from dev.storvsc.X. if deviceid matches
+    """
+    dev.storvsc.1.%pnpinfo:
+        classid=32412632-86cb-44a2-9b5c-50d1417354f5
+        deviceid=00000000-0001-8899-0000-000000000000
+    """
+    for line in sysctl_out.splitlines():
+        if re.search(r"pnpinfo", line):
+            fields = line.split()
+            if len(fields) >= 3:
+                columns = fields[2].split('=')
+                if (len(columns) >= 2 and
+                        columns[0] == "deviceid" and
+                        columns[1].startswith(deviceid)):
+                    comps = fields[0].split('.')
+                    return comps[2]
+    return None
+
+
+def find_busdev_from_disk(camcontrol_out, disk_drv):
+    # find the scbusX from 'camcontrol devlist -b' output
+    # if disk_drv matches the specified disk driver, i.e. blkvsc1
+    """
+    scbus0 on ata0 bus 0
+    scbus1 on ata1 bus 0
+    scbus2 on blkvsc0 bus 0
+    scbus3 on blkvsc1 bus 0
+    scbus4 on storvsc2 bus 0
+    scbus5 on storvsc3 bus 0
+    scbus-1 on xpt0 bus 0
+    """
+    for line in camcontrol_out.splitlines():
+        if re.search(disk_drv, line):
+            items = line.split()
+            return items[0]
+    return None
+
+
+def find_dev_from_busdev(camcontrol_out, busdev):
+    # find the daX from 'camcontrol devlist' output
+    # if busdev matches the specified value, i.e. 'scbus2'
+    """
+    <Msft Virtual CD/ROM 1.0>          at scbus1 target 0 lun 0 (cd0,pass0)
+    <Msft Virtual Disk 1.0>            at scbus2 target 0 lun 0 (da0,pass1)
+    <Msft Virtual Disk 1.0>            at scbus3 target 1 lun 0 (da1,pass2)
+    """
+    for line in camcontrol_out.splitlines():
+        if re.search(busdev, line):
+            items = line.split('(')
+            if len(items) == 2:
+                dev_pass = items[1].split(',')
+                return dev_pass[0]
+    return None
+
+
+def get_dev_storvsc_sysctl():
+    try:
+        sysctl_out, err = util.subp(['sysctl', 'dev.storvsc'])
+    except util.ProcessExecutionError:
+        LOG.debug("Fail to execute sysctl dev.storvsc")
+        return None
+    return sysctl_out
+
+
+def get_camcontrol_dev_bus():
+    try:
+        camcontrol_b_out, err = util.subp(['camcontrol', 'devlist', '-b'])
+    except util.ProcessExecutionError:
+        LOG.debug("Fail to execute camcontrol devlist -b")
+        return None
+    return camcontrol_b_out
+
+
+def get_camcontrol_dev():
+    try:
+        camcontrol_out, err = util.subp(['camcontrol', 'devlist'])
+    except util.ProcessExecutionError:
+        LOG.debug("Fail to execute camcontrol devlist")
+        return None
+    return camcontrol_out
+
+
+def get_resource_disk_on_freebsd(port_id):
+    g0 = "00000000"
+    if port_id > 1:
+        g0 = "00000001"
+        port_id = port_id - 2
+    g1 = "000" + str(port_id)
+    g0g1 = "{0}-{1}".format(g0, g1)
+    """
+    search 'X' from
+       'dev.storvsc.X.%pnpinfo:
+           classid=32412632-86cb-44a2-9b5c-50d1417354f5
+           deviceid=00000000-0001-8899-0000-000000000000'
+    """
+    sysctl_out = get_dev_storvsc_sysctl()
+
+    storvscid = find_storvscid_from_sysctl_pnpinfo(sysctl_out, g0g1)
+    if not storvscid:
+        LOG.debug("Fail to find storvsc id from sysctl")
+        return None
+
+    camcontrol_b_out = get_camcontrol_dev_bus()
+    camcontrol_out = get_camcontrol_dev()
+    # try to find /dev/XX from 'blkvsc' device
+    blkvsc = "blkvsc{0}".format(storvscid)
+    scbusx = find_busdev_from_disk(camcontrol_b_out, blkvsc)
+    if scbusx:
+        devname = find_dev_from_busdev(camcontrol_out, scbusx)
+        if devname is None:
+            LOG.debug("Fail to find /dev/daX")
+            return None
+        return devname
+    # try to find /dev/XX from 'storvsc' device
+    storvsc = "storvsc{0}".format(storvscid)
+    scbusx = find_busdev_from_disk(camcontrol_b_out, storvsc)
+    if scbusx:
+        devname = find_dev_from_busdev(camcontrol_out, scbusx)
+        if devname is None:
+            LOG.debug("Fail to find /dev/daX")
+            return None
+        return devname
+    return None
+
+# update the FreeBSD specific information
+if util.is_FreeBSD():
+    DEFAULT_PRIMARY_NIC = 'hn0'
+    LEASE_FILE = '/var/db/dhclient.leases.hn0'
+    DEFAULT_FS = 'freebsd-ufs'
+    res_disk = get_resource_disk_on_freebsd(1)
+    if res_disk is not None:
+        LOG.debug("resource disk is not None")
+        RESOURCE_DISK_PATH = "/dev/" + res_disk
+    else:
+        LOG.debug("resource disk is None")
 
 BUILTIN_DS_CONFIG = {
     'agent_command': AGENT_START_BUILTIN,
     'data_dir': "/var/lib/waagent",
     'set_hostname': True,
     'hostname_bounce': {
-        'interface': 'eth0',
+        'interface': DEFAULT_PRIMARY_NIC,
         'policy': True,
         'command': BOUNCE_COMMAND,
         'hostname_command': 'hostname',
     },
     'disk_aliases': {'ephemeral0': RESOURCE_DISK_PATH},
-    'dhclient_lease_file': '/var/lib/dhcp/dhclient.eth0.leases',
+    'dhclient_lease_file': LEASE_FILE,
 }
 
 BUILTIN_CLOUD_CONFIG = {
@@ -53,7 +195,7 @@ BUILTIN_CLOUD_CONFIG = {
                        'layout': [100],
                        'overwrite': True},
     },
-    'fs_setup': [{'filesystem': 'ext4',
+    'fs_setup': [{'filesystem': DEFAULT_FS,
                   'device': 'ephemeral0.1',
                   'replace_fs': 'ntfs'}],
 }
@@ -190,7 +332,11 @@ class DataSourceAzureNet(sources.DataSource):
         for cdev in candidates:
             try:
                 if cdev.startswith("/dev/"):
-                    ret = util.mount_cb(cdev, load_azure_ds_dir)
+                    if util.is_FreeBSD():
+                        ret = util.mount_cb(cdev, load_azure_ds_dir,
+                                            mtype="udf", sync=False)
+                    else:
+                        ret = util.mount_cb(cdev, load_azure_ds_dir)
                 else:
                     ret = load_azure_ds_dir(cdev)
 
@@ -218,11 +364,12 @@ class DataSourceAzureNet(sources.DataSource):
             LOG.debug("using files cached in %s", ddir)
 
         # azure / hyper-v provides random data here
-        seed = util.load_file("/sys/firmware/acpi/tables/OEM0",
-                              quiet=True, decode=False)
-        if seed:
-            self.metadata['random_seed'] = seed
-
+        if not util.is_FreeBSD():
+            seed = util.load_file("/sys/firmware/acpi/tables/OEM0",
+                                  quiet=True, decode=False)
+            if seed:
+                self.metadata['random_seed'] = seed
+        # TODO. find the seed on FreeBSD platform
         # now update ds_cfg to reflect contents pass in config
         user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {})
         self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg])
@@ -633,8 +780,19 @@ def encrypt_pass(password, salt_id="$6$"):
 def list_possible_azure_ds_devs():
     # return a sorted list of devices that might have a azure datasource
     devlist = []
-    for fstype in ("iso9660", "udf"):
-        devlist.extend(util.find_devs_with("TYPE=%s" % fstype))
+    if util.is_FreeBSD():
+        cdrom_dev = "/dev/cd0"
+        try:
+            util.subp(["mount", "-o", "ro", "-t", "udf", cdrom_dev,
+                       "/mnt/cdrom/secure"])
+        except util.ProcessExecutionError:
+            LOG.debug("Fail to mount cd")
+            return devlist
+        util.subp(["umount", "/mnt/cdrom/secure"])
+        devlist.append(cdrom_dev)
+    else:
+        for fstype in ("iso9660", "udf"):
+            devlist.extend(util.find_devs_with("TYPE=%s" % fstype))
 
     devlist.sort(reverse=True)
     return devlist
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 6e01aa47..e22409d1 100644
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -29,6 +29,14 @@ def cd(newdir):
         os.chdir(prevdir)
 
 
+def _get_dhcp_endpoint_option_name():
+    if util.is_FreeBSD():
+        azure_endpoint = "option-245"
+    else:
+        azure_endpoint = "unknown-245"
+    return azure_endpoint
+
+
 class AzureEndpointHttpClient(object):
 
     headers = {
@@ -235,8 +243,9 @@ class WALinuxAgentShim(object):
         leases = []
         content = util.load_file(fallback_lease_file)
         LOG.debug("content is %s", content)
+        option_name = _get_dhcp_endpoint_option_name()
         for line in content.splitlines():
-            if 'unknown-245' in line:
+            if option_name in line:
                 # Example line from Ubuntu
                 # option unknown-245 a8:3f:81:10;
                 leases.append(line.strip(' ').split(' ', 2)[-1].strip(';\n"'))