Add Netbsd support (#62)

Add support for the NetBSD Operating System.                                    
                                                                                
Features in this branch:                                                        
* Add BSD distro parent class from which NetBSD and FreeBSD can                 
  specialize                                                                    
* Add *bsd util functions to cloudinit.net and cloudinit.net.bsd_utils          
* subclass cloudinit.distro.freebsd.Distro from bsd.Distro                      
* Add new cloudinit.distro.netbsd and cloudinit.net.renderer for                
  netbsd                                                                        
* Add lru_cached util.is_NetBSD functions                                       
* Add NetBSD detection for ConfigDrive and NoCloud datasources                  
                                                                                
This branch has been tested with:                                               
                                                                                
- NoCloud and OpenStack (with and without config-drive)                         
- NetBSD 8.1. and 9.0                                                           
- FreeBSD 11.2 and 12.1                                                         
- Python 3.7 only, because of the dependency oncrypt.METHOD_BLOWFISH.           
  This version is available in NetBSD 7, 8 and 9 anyway

4 files changed, 309 insertions, 118 deletions

diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py
new file mode 100644
index 00000000..e9b84edc
--- /dev/null
+++ b/cloudinit/distros/bsd.py
@@ -0,0 +1,111 @@
+import platform
+
+from cloudinit import distros
+from cloudinit.distros import bsd_utils
+from cloudinit import helpers
+from cloudinit import log as logging
+from cloudinit import net
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+class BSD(distros.Distro):
+    hostname_conf_fn = '/etc/rc.conf'
+    rc_conf_fn = "/etc/rc.conf"
+
+    # Set in BSD distro subclasses
+    group_add_cmd_prefix = []
+    pkg_cmd_install_prefix = []
+    pkg_cmd_remove_prefix = []
+
+    def __init__(self, name, cfg, paths):
+        super().__init__(name, cfg, paths)
+        # This will be used to restrict certain
+        # calls from repeatly happening (when they
+        # should only happen say once per instance...)
+        self._runner = helpers.Runners(paths)
+        cfg['ssh_svcname'] = 'sshd'
+        self.osfamily = platform.system().lower()
+
+    def _read_system_hostname(self):
+        sys_hostname = self._read_hostname(self.hostname_conf_fn)
+        return (self.hostname_conf_fn, sys_hostname)
+
+    def _read_hostname(self, filename, default=None):
+        return bsd_utils.get_rc_config_value('hostname')
+
+    def _get_add_member_to_group_cmd(self, member_name, group_name):
+        raise NotImplementedError('Return list cmd to add member to group')
+
+    def _write_hostname(self, hostname, filename):
+        bsd_utils.set_rc_config_value('hostname', hostname, fn='/etc/rc.conf')
+
+    def create_group(self, name, members=None):
+        if util.is_group(name):
+            LOG.warning("Skipping creation of existing group '%s'", name)
+        else:
+            group_add_cmd = self.group_add_cmd_prefix + [name]
+            try:
+                util.subp(group_add_cmd)
+                LOG.info("Created new group %s", name)
+            except Exception:
+                util.logexc(LOG, "Failed to create group %s", name)
+
+        if not members:
+            members = []
+        for member in members:
+            if not util.is_user(member):
+                LOG.warning("Unable to add group member '%s' to group '%s'"
+                            "; user does not exist.", member, name)
+                continue
+            try:
+                util.subp(self._get_add_member_to_group_cmd(member, name))
+                LOG.info("Added user '%s' to group '%s'", member, name)
+            except Exception:
+                util.logexc(LOG, "Failed to add user '%s' to group '%s'",
+                            member, name)
+
+    def generate_fallback_config(self):
+        nconf = {'config': [], 'version': 1}
+        for mac, name in net.get_interfaces_by_mac().items():
+            nconf['config'].append(
+                {'type': 'physical', 'name': name,
+                 'mac_address': mac, 'subnets': [{'type': 'dhcp'}]})
+        return nconf
+
+    def install_packages(self, pkglist):
+        self.update_package_sources()
+        self.package_command('install', pkgs=pkglist)
+
+    def _get_pkg_cmd_environ(self):
+        """Return environment vars used in *BSD package_command operations"""
+        raise NotImplementedError('BSD subclasses return a dict of env vars')
+
+    def package_command(self, command, args=None, pkgs=None):
+        if pkgs is None:
+            pkgs = []
+
+        # TODO neither freebsd nor netbsd handles a command 'upgrade'
+        # provided by cloudinit/config/cc_package_update_upgrade_install.py
+        if command == 'install':
+            cmd = self.pkg_cmd_install_prefix
+        elif command == 'remove':
+            cmd = self.pkg_cmd_remove_prefix
+
+        if args and isinstance(args, str):
+            cmd.append(args)
+        elif args and isinstance(args, list):
+            cmd.extend(args)
+
+        pkglist = util.expand_package_list('%s-%s', pkgs)
+        cmd.extend(pkglist)
+
+        # Allow the output of this to flow outwards (ie not be captured)
+        util.subp(cmd, env=self._get_pkg_cmd_environ(), capture=False)
+
+    def _write_network_config(self, netconfig):
+        return self._supported_write_network_config(netconfig)
+
+    def set_timezone(self, tz):
+        distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz))
diff --git a/cloudinit/distros/bsd_utils.py b/cloudinit/distros/bsd_utils.py
new file mode 100644
index 00000000..079d0d53
--- /dev/null
+++ b/cloudinit/distros/bsd_utils.py
@@ -0,0 +1,50 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import shlex
+
+from cloudinit import util
+
+# On NetBSD, /etc/rc.conf comes with a if block:
+#   if [ -r /etc/defaults/rc.conf ]; then
+# as a consequence, the file is not a regular key/value list
+# anymore and we cannot use cloudinit.distros.parsers.sys_conf
+# The module comes with a more naive parser, but is able to
+# preserve these if blocks.
+
+
+def _unquote(value):
+    if value[0] == value[-1] and value[0] in ['"', "'"]:
+        return value[1:-1]
+    return value
+
+
+def get_rc_config_value(key, fn='/etc/rc.conf'):
+    key_prefix = '{}='.format(key)
+    for line in util.load_file(fn).splitlines():
+        if line.startswith(key_prefix):
+            value = line.replace(key_prefix, '')
+            return _unquote(value)
+
+
+def set_rc_config_value(key, value, fn='/etc/rc.conf'):
+    lines = []
+    done = False
+    value = shlex.quote(value)
+    original_content = util.load_file(fn)
+    for line in original_content.splitlines():
+        if '=' in line:
+            k, v = line.split('=', 1)
+            if k == key:
+                v = value
+                done = True
+            lines.append('='.join([k, v]))
+        else:
+            lines.append(line)
+    if not done:
+        lines.append('='.join([key, value]))
+    new_content = '\n'.join(lines) + '\n'
+    if new_content != original_content:
+        util.write_file(fn, new_content)
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py
index 026d1142..a775ae51 100644
--- a/cloudinit/distros/freebsd.py
+++ b/cloudinit/distros/freebsd.py
@@ -8,34 +8,22 @@ import os
 import re
 from io import StringIO
 
-from cloudinit import distros
-from cloudinit import helpers
+import cloudinit.distros.bsd
 from cloudinit import log as logging
-from cloudinit import net
-from cloudinit import ssh_util
 from cloudinit import util
-from cloudinit.distros import rhel_util
 from cloudinit.settings import PER_INSTANCE
 
 LOG = logging.getLogger(__name__)
 
 
-class Distro(distros.Distro):
+class Distro(cloudinit.distros.bsd.BSD):
     usr_lib_exec = '/usr/local/lib'
-    rc_conf_fn = "/etc/rc.conf"
     login_conf_fn = '/etc/login.conf'
     login_conf_fn_bak = '/etc/login.conf.orig'
     ci_sudoers_fn = '/usr/local/etc/sudoers.d/90-cloud-init-users'
-    hostname_conf_fn = '/etc/rc.conf'
-
-    def __init__(self, name, cfg, paths):
-        distros.Distro.__init__(self, name, cfg, paths)
-        # This will be used to restrict certain
-        # calls from repeatly happening (when they
-        # should only happen say once per instance...)
-        self._runner = helpers.Runners(paths)
-        self.osfamily = 'freebsd'
-        cfg['ssh_svcname'] = 'sshd'
+    group_add_cmd_prefix = ['pw', 'group', 'add']
+    pkg_cmd_install_prefix = ["pkg", "install"]
+    pkg_cmd_remove_prefix = ["pkg", "remove"]
 
     def _select_hostname(self, hostname, fqdn):
         # Should be FQDN if available. See rc.conf(5) in FreeBSD
@@ -43,45 +31,8 @@ class Distro(distros.Distro):
             return fqdn
         return hostname
 
-    def _read_system_hostname(self):
-        sys_hostname = self._read_hostname(self.hostname_conf_fn)
-        return (self.hostname_conf_fn, sys_hostname)
-
-    def _read_hostname(self, filename, default=None):
-        (_exists, contents) = rhel_util.read_sysconfig_file(filename)
-        if contents.get('hostname'):
-            return contents['hostname']
-        else:
-            return default
-
-    def _write_hostname(self, hostname, filename):
-        rhel_util.update_sysconfig_file(filename, {'hostname': hostname})
-
-    def create_group(self, name, members):
-        group_add_cmd = ['pw', 'group', 'add', name]
-        if util.is_group(name):
-            LOG.warning("Skipping creation of existing group '%s'", name)
-        else:
-            try:
-                util.subp(group_add_cmd)
-                LOG.info("Created new group %s", name)
-            except Exception:
-                util.logexc(LOG, "Failed to create group %s", name)
-                raise
-        if not members:
-            members = []
-
-        for member in members:
-            if not util.is_user(member):
-                LOG.warning("Unable to add group member '%s' to group '%s'"
-                            "; user does not exist.", member, name)
-                continue
-            try:
-                util.subp(['pw', 'usermod', '-n', name, '-G', member])
-                LOG.info("Added user '%s' to group '%s'", member, name)
-            except Exception:
-                util.logexc(LOG, "Failed to add user '%s' to group '%s'",
-                            member, name)
+    def _get_add_member_to_group_cmd(self, member_name, group_name):
+        return ['pw', 'usermod', '-n', member_name, '-G', group_name]
 
     def add_user(self, name, **kwargs):
         if util.is_user(name):
@@ -162,40 +113,8 @@ class Distro(distros.Distro):
             util.logexc(LOG, "Failed to lock user %s", name)
             raise
 
-    def create_user(self, name, **kwargs):
-        self.add_user(name, **kwargs)
-
-        # Set password if plain-text password provided and non-empty
-        if 'plain_text_passwd' in kwargs and kwargs['plain_text_passwd']:
-            self.set_passwd(name, kwargs['plain_text_passwd'])
-
-        # Default locking down the account. 'lock_passwd' defaults to True.
-        # lock account unless lock_password is False.
-        if kwargs.get('lock_passwd', True):
-            self.lock_passwd(name)
-
-        # Configure sudo access
-        if 'sudo' in kwargs and kwargs['sudo'] is not False:
-            self.write_sudo_rules(name, kwargs['sudo'])
-
-        # Import SSH keys
-        if 'ssh_authorized_keys' in kwargs:
-            keys = set(kwargs['ssh_authorized_keys']) or []
-            ssh_util.setup_user_keys(keys, name, options=None)
-
-    def generate_fallback_config(self):
-        nconf = {'config': [], 'version': 1}
-        for mac, name in net.get_interfaces_by_mac().items():
-            nconf['config'].append(
-                {'type': 'physical', 'name': name,
-                 'mac_address': mac, 'subnets': [{'type': 'dhcp'}]})
-        return nconf
-
-    def _write_network_config(self, netconfig):
-        return self._supported_write_network_config(netconfig)
-
     def apply_locale(self, locale, out_fn=None):
-        # Adjust the locals value to the new value
+        # Adjust the locales value to the new value
         newconf = StringIO()
         for line in util.load_file(self.login_conf_fn).splitlines():
             newconf.write(re.sub(r'^default:',
@@ -225,39 +144,17 @@ class Distro(distros.Distro):
         # /etc/rc.conf a line with the following format:
         #    ifconfig_OLDNAME_name=NEWNAME
         # FreeBSD network script will rename the interface automatically.
-        return
-
-    def install_packages(self, pkglist):
-        self.update_package_sources()
-        self.package_command('install', pkgs=pkglist)
-
-    def package_command(self, command, args=None, pkgs=None):
-        if pkgs is None:
-            pkgs = []
+        pass
 
+    def _get_pkg_cmd_environ(self):
+        """Return environment vars used in *BSD package_command operations"""
         e = os.environ.copy()
         e['ASSUME_ALWAYS_YES'] = 'YES'
-
-        cmd = ['pkg']
-        if args and isinstance(args, str):
-            cmd.append(args)
-        elif args and isinstance(args, list):
-            cmd.extend(args)
-
-        if command:
-            cmd.append(command)
-
-        pkglist = util.expand_package_list('%s-%s', pkgs)
-        cmd.extend(pkglist)
-
-        # Allow the output of this to flow outwards (ie not be captured)
-        util.subp(cmd, env=e, capture=False)
-
-    def set_timezone(self, tz):
-        distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz))
+        return e
 
     def update_package_sources(self):
-        self._runner.run("update-sources", self.package_command,
-                         ["update"], freq=PER_INSTANCE)
+        self._runner.run(
+            "update-sources", self.package_command,
+            ["update"], freq=PER_INSTANCE)
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py
new file mode 100644
index 00000000..353eb671
--- /dev/null
+++ b/cloudinit/distros/netbsd.py
@@ -0,0 +1,133 @@
+# Copyright (C) 2019-2020 Gonéri Le Bouder
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import crypt
+import os
+import platform
+import six
+
+import cloudinit.distros.bsd
+from cloudinit import log as logging
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+class Distro(cloudinit.distros.bsd.BSD):
+    ci_sudoers_fn = '/usr/pkg/etc/sudoers.d/90-cloud-init-users'
+
+    group_add_cmd_prefix = ["groupadd"]
+    pkg_cmd_install_prefix = ["pkg_add", "-U"]
+    pkg_cmd_remove_prefix = ['pkg_delete']
+
+    def _get_add_member_to_group_cmd(self, member_name, group_name):
+        return ['usermod', '-G', group_name, member_name]
+
+    def add_user(self, name, **kwargs):
+        if util.is_user(name):
+            LOG.info("User %s already exists, skipping.", name)
+            return False
+
+        adduser_cmd = ['useradd']
+        log_adduser_cmd = ['useradd']
+
+        adduser_opts = {
+            "homedir": '-d',
+            "gecos": '-c',
+            "primary_group": '-g',
+            "groups": '-G',
+            "shell": '-s',
+        }
+        adduser_flags = {
+            "no_user_group": '--no-user-group',
+            "system": '--system',
+            "no_log_init": '--no-log-init',
+        }
+
+        for key, val in kwargs.items():
+            if (key in adduser_opts and val and
+               isinstance(val, six.string_types)):
+                adduser_cmd.extend([adduser_opts[key], val])
+
+            elif key in adduser_flags and val:
+                adduser_cmd.append(adduser_flags[key])
+                log_adduser_cmd.append(adduser_flags[key])
+
+        if 'no_create_home' not in kwargs or 'system' not in kwargs:
+            adduser_cmd += ['-m']
+            log_adduser_cmd += ['-m']
+
+        adduser_cmd += [name]
+        log_adduser_cmd += [name]
+
+        # Run the command
+        LOG.info("Adding user %s", name)
+        try:
+            util.subp(adduser_cmd, logstring=log_adduser_cmd)
+        except Exception:
+            util.logexc(LOG, "Failed to create user %s", name)
+            raise
+        # Set the password if it is provided
+        # For security consideration, only hashed passwd is assumed
+        passwd_val = kwargs.get('passwd', None)
+        if passwd_val is not None:
+            self.set_passwd(name, passwd_val, hashed=True)
+
+    def set_passwd(self, user, passwd, hashed=False):
+        if hashed:
+            hashed_pw = passwd
+        elif not hasattr(crypt, 'METHOD_BLOWFISH'):
+            # crypt.METHOD_BLOWFISH comes with Python 3.7 which is available
+            # on NetBSD 7 and 8.
+            LOG.error((
+                'Cannot set non-encrypted password for user %s. '
+                'Python >= 3.7 is required.'), user)
+            return
+        else:
+            method = crypt.METHOD_BLOWFISH  # pylint: disable=E1101
+            hashed_pw = crypt.crypt(
+                    passwd,
+                    crypt.mksalt(method))
+
+        try:
+            util.subp(['usermod', '-C', 'no', '-p', hashed_pw, user])
+        except Exception:
+            util.logexc(LOG, "Failed to set password for %s", user)
+            raise
+
+    def force_passwd_change(self, user):
+        try:
+            util.subp(['usermod', '-F', user])
+        except Exception:
+            util.logexc(LOG, "Failed to set pw expiration for %s", user)
+            raise
+
+    def lock_passwd(self, name):
+        try:
+            util.subp(['usermod', '-C', 'yes', name])
+        except Exception:
+            util.logexc(LOG, "Failed to lock user %s", name)
+            raise
+
+    def apply_locale(self, locale, out_fn=None):
+        LOG.debug('Cannot set the locale.')
+
+    def apply_network_config_names(self, netconfig):
+        LOG.debug('NetBSD cannot rename network interface.')
+
+    def _get_pkg_cmd_environ(self):
+        """Return environment vars used in *BSD package_command operations"""
+        os_release = platform.release()
+        os_arch = platform.machine()
+        e = os.environ.copy()
+        e['PKG_PATH'] = (
+                'http://cdn.netbsd.org/pub/pkgsrc/'
+                'packages/NetBSD/%s/%s/All') % (os_arch, os_release)
+        return e
+
+    def update_package_sources(self):
+        pass
+
+
+# vi: ts=4 expandtab