From 94838def772349387e16cc642b3642020e22deda Mon Sep 17 00:00:00 2001
From: Gonéri Le Bouder <goneri@lebouder.net>
Date: Thu, 12 Mar 2020 14:37:08 -0400
Subject: Add Netbsd support (#62)

Add support for the NetBSD Operating System.

Features in this branch:
* Add BSD distro parent class from which NetBSD and FreeBSD can
  specialize
* Add *bsd util functions to cloudinit.net and cloudinit.net.bsd_utils
* subclass cloudinit.distro.freebsd.Distro from bsd.Distro
* Add new cloudinit.distro.netbsd and cloudinit.net.renderer for
  netbsd
* Add lru_cached util.is_NetBSD functions
* Add NetBSD detection for ConfigDrive and NoCloud datasources

This branch has been tested with:

- NoCloud and OpenStack (with and without config-drive)
- NetBSD 8.1. and 9.0
- FreeBSD 11.2 and 12.1
- Python 3.7 only, because of the dependency oncrypt.METHOD_BLOWFISH.
  This version is available in NetBSD 7, 8 and 9 anyway
---
 cloudinit/distros/bsd.py       | 111 ++++++++++++++++++++++++++++++++++
 cloudinit/distros/bsd_utils.py |  50 ++++++++++++++++
 cloudinit/distros/freebsd.py   | 133 +++++------------------------------------
 cloudinit/distros/netbsd.py    | 133 +++++++++++++++++++++++++++++++++++++++++
 4 files changed, 309 insertions(+), 118 deletions(-)
 create mode 100644 cloudinit/distros/bsd.py
 create mode 100644 cloudinit/distros/bsd_utils.py
 create mode 100644 cloudinit/distros/netbsd.py

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py
new file mode 100644
index 00000000..e9b84edc
--- /dev/null
+++ b/cloudinit/distros/bsd.py
@@ -0,0 +1,111 @@
+import platform
+
+from cloudinit import distros
+from cloudinit.distros import bsd_utils
+from cloudinit import helpers
+from cloudinit import log as logging
+from cloudinit import net
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+class BSD(distros.Distro):
+    hostname_conf_fn = '/etc/rc.conf'
+    rc_conf_fn = "/etc/rc.conf"
+
+    # Set in BSD distro subclasses
+    group_add_cmd_prefix = []
+    pkg_cmd_install_prefix = []
+    pkg_cmd_remove_prefix = []
+
+    def __init__(self, name, cfg, paths):
+        super().__init__(name, cfg, paths)
+        # This will be used to restrict certain
+        # calls from repeatly happening (when they
+        # should only happen say once per instance...)
+        self._runner = helpers.Runners(paths)
+        cfg['ssh_svcname'] = 'sshd'
+        self.osfamily = platform.system().lower()
+
+    def _read_system_hostname(self):
+        sys_hostname = self._read_hostname(self.hostname_conf_fn)
+        return (self.hostname_conf_fn, sys_hostname)
+
+    def _read_hostname(self, filename, default=None):
+        return bsd_utils.get_rc_config_value('hostname')
+
+    def _get_add_member_to_group_cmd(self, member_name, group_name):
+        raise NotImplementedError('Return list cmd to add member to group')
+
+    def _write_hostname(self, hostname, filename):
+        bsd_utils.set_rc_config_value('hostname', hostname, fn='/etc/rc.conf')
+
+    def create_group(self, name, members=None):
+        if util.is_group(name):
+            LOG.warning("Skipping creation of existing group '%s'", name)
+        else:
+            group_add_cmd = self.group_add_cmd_prefix + [name]
+            try:
+                util.subp(group_add_cmd)
+                LOG.info("Created new group %s", name)
+            except Exception:
+                util.logexc(LOG, "Failed to create group %s", name)
+
+        if not members:
+            members = []
+        for member in members:
+            if not util.is_user(member):
+                LOG.warning("Unable to add group member '%s' to group '%s'"
+                            "; user does not exist.", member, name)
+                continue
+            try:
+                util.subp(self._get_add_member_to_group_cmd(member, name))
+                LOG.info("Added user '%s' to group '%s'", member, name)
+            except Exception:
+                util.logexc(LOG, "Failed to add user '%s' to group '%s'",
+                            member, name)
+
+    def generate_fallback_config(self):
+        nconf = {'config': [], 'version': 1}
+        for mac, name in net.get_interfaces_by_mac().items():
+            nconf['config'].append(
+                {'type': 'physical', 'name': name,
+                 'mac_address': mac, 'subnets': [{'type': 'dhcp'}]})
+        return nconf
+
+    def install_packages(self, pkglist):
+        self.update_package_sources()
+        self.package_command('install', pkgs=pkglist)
+
+    def _get_pkg_cmd_environ(self):
+        """Return environment vars used in *BSD package_command operations"""
+        raise NotImplementedError('BSD subclasses return a dict of env vars')
+
+    def package_command(self, command, args=None, pkgs=None):
+        if pkgs is None:
+            pkgs = []
+
+        # TODO neither freebsd nor netbsd handles a command 'upgrade'
+        # provided by cloudinit/config/cc_package_update_upgrade_install.py
+        if command == 'install':
+            cmd = self.pkg_cmd_install_prefix
+        elif command == 'remove':
+            cmd = self.pkg_cmd_remove_prefix
+
+        if args and isinstance(args, str):
+            cmd.append(args)
+        elif args and isinstance(args, list):
+            cmd.extend(args)
+
+        pkglist = util.expand_package_list('%s-%s', pkgs)
+        cmd.extend(pkglist)
+
+        # Allow the output of this to flow outwards (ie not be captured)
+        util.subp(cmd, env=self._get_pkg_cmd_environ(), capture=False)
+
+    def _write_network_config(self, netconfig):
+        return self._supported_write_network_config(netconfig)
+
+    def set_timezone(self, tz):
+        distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz))
diff --git a/cloudinit/distros/bsd_utils.py b/cloudinit/distros/bsd_utils.py
new file mode 100644
index 00000000..079d0d53
--- /dev/null
+++ b/cloudinit/distros/bsd_utils.py
@@ -0,0 +1,50 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import shlex
+
+from cloudinit import util
+
+# On NetBSD, /etc/rc.conf comes with a if block:
+#   if [ -r /etc/defaults/rc.conf ]; then
+# as a consequence, the file is not a regular key/value list
+# anymore and we cannot use cloudinit.distros.parsers.sys_conf
+# The module comes with a more naive parser, but is able to
+# preserve these if blocks.
+
+
+def _unquote(value):
+    if value[0] == value[-1] and value[0] in ['"', "'"]:
+        return value[1:-1]
+    return value
+
+
+def get_rc_config_value(key, fn='/etc/rc.conf'):
+    key_prefix = '{}='.format(key)
+    for line in util.load_file(fn).splitlines():
+        if line.startswith(key_prefix):
+            value = line.replace(key_prefix, '')
+            return _unquote(value)
+
+
+def set_rc_config_value(key, value, fn='/etc/rc.conf'):
+    lines = []
+    done = False
+    value = shlex.quote(value)
+    original_content = util.load_file(fn)
+    for line in original_content.splitlines():
+        if '=' in line:
+            k, v = line.split('=', 1)
+            if k == key:
+                v = value
+                done = True
+            lines.append('='.join([k, v]))
+        else:
+            lines.append(line)
+    if not done:
+        lines.append('='.join([key, value]))
+    new_content = '\n'.join(lines) + '\n'
+    if new_content != original_content:
+        util.write_file(fn, new_content)
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py
index 026d1142..a775ae51 100644
--- a/cloudinit/distros/freebsd.py
+++ b/cloudinit/distros/freebsd.py
@@ -8,34 +8,22 @@ import os
 import re
 from io import StringIO
 
-from cloudinit import distros
-from cloudinit import helpers
+import cloudinit.distros.bsd
 from cloudinit import log as logging
-from cloudinit import net
-from cloudinit import ssh_util
 from cloudinit import util
-from cloudinit.distros import rhel_util
 from cloudinit.settings import PER_INSTANCE
 
 LOG = logging.getLogger(__name__)
 
 
-class Distro(distros.Distro):
+class Distro(cloudinit.distros.bsd.BSD):
     usr_lib_exec = '/usr/local/lib'
-    rc_conf_fn = "/etc/rc.conf"
     login_conf_fn = '/etc/login.conf'
     login_conf_fn_bak = '/etc/login.conf.orig'
     ci_sudoers_fn = '/usr/local/etc/sudoers.d/90-cloud-init-users'
-    hostname_conf_fn = '/etc/rc.conf'
-
-    def __init__(self, name, cfg, paths):
-        distros.Distro.__init__(self, name, cfg, paths)
-        # This will be used to restrict certain
-        # calls from repeatly happening (when they
-        # should only happen say once per instance...)
-        self._runner = helpers.Runners(paths)
-        self.osfamily = 'freebsd'
-        cfg['ssh_svcname'] = 'sshd'
+    group_add_cmd_prefix = ['pw', 'group', 'add']
+    pkg_cmd_install_prefix = ["pkg", "install"]
+    pkg_cmd_remove_prefix = ["pkg", "remove"]
 
     def _select_hostname(self, hostname, fqdn):
         # Should be FQDN if available. See rc.conf(5) in FreeBSD
@@ -43,45 +31,8 @@ class Distro(distros.Distro):
             return fqdn
         return hostname
 
-    def _read_system_hostname(self):
-        sys_hostname = self._read_hostname(self.hostname_conf_fn)
-        return (self.hostname_conf_fn, sys_hostname)
-
-    def _read_hostname(self, filename, default=None):
-        (_exists, contents) = rhel_util.read_sysconfig_file(filename)
-        if contents.get('hostname'):
-            return contents['hostname']
-        else:
-            return default
-
-    def _write_hostname(self, hostname, filename):
-        rhel_util.update_sysconfig_file(filename, {'hostname': hostname})
-
-    def create_group(self, name, members):
-        group_add_cmd = ['pw', 'group', 'add', name]
-        if util.is_group(name):
-            LOG.warning("Skipping creation of existing group '%s'", name)
-        else:
-            try:
-                util.subp(group_add_cmd)
-                LOG.info("Created new group %s", name)
-            except Exception:
-                util.logexc(LOG, "Failed to create group %s", name)
-                raise
-        if not members:
-            members = []
-
-        for member in members:
-            if not util.is_user(member):
-                LOG.warning("Unable to add group member '%s' to group '%s'"
-                            "; user does not exist.", member, name)
-                continue
-            try:
-                util.subp(['pw', 'usermod', '-n', name, '-G', member])
-                LOG.info("Added user '%s' to group '%s'", member, name)
-            except Exception:
-                util.logexc(LOG, "Failed to add user '%s' to group '%s'",
-                            member, name)
+    def _get_add_member_to_group_cmd(self, member_name, group_name):
+        return ['pw', 'usermod', '-n', member_name, '-G', group_name]
 
     def add_user(self, name, **kwargs):
         if util.is_user(name):
@@ -162,40 +113,8 @@ class Distro(distros.Distro):
             util.logexc(LOG, "Failed to lock user %s", name)
             raise
 
-    def create_user(self, name, **kwargs):
-        self.add_user(name, **kwargs)
-
-        # Set password if plain-text password provided and non-empty
-        if 'plain_text_passwd' in kwargs and kwargs['plain_text_passwd']:
-            self.set_passwd(name, kwargs['plain_text_passwd'])
-
-        # Default locking down the account. 'lock_passwd' defaults to True.
-        # lock account unless lock_password is False.
-        if kwargs.get('lock_passwd', True):
-            self.lock_passwd(name)
-
-        # Configure sudo access
-        if 'sudo' in kwargs and kwargs['sudo'] is not False:
-            self.write_sudo_rules(name, kwargs['sudo'])
-
-        # Import SSH keys
-        if 'ssh_authorized_keys' in kwargs:
-            keys = set(kwargs['ssh_authorized_keys']) or []
-            ssh_util.setup_user_keys(keys, name, options=None)
-
-    def generate_fallback_config(self):
-        nconf = {'config': [], 'version': 1}
-        for mac, name in net.get_interfaces_by_mac().items():
-            nconf['config'].append(
-                {'type': 'physical', 'name': name,
-                 'mac_address': mac, 'subnets': [{'type': 'dhcp'}]})
-        return nconf
-
-    def _write_network_config(self, netconfig):
-        return self._supported_write_network_config(netconfig)
-
     def apply_locale(self, locale, out_fn=None):
-        # Adjust the locals value to the new value
+        # Adjust the locales value to the new value
         newconf = StringIO()
         for line in util.load_file(self.login_conf_fn).splitlines():
             newconf.write(re.sub(r'^default:',
@@ -225,39 +144,17 @@ class Distro(distros.Distro):
         # /etc/rc.conf a line with the following format:
         #    ifconfig_OLDNAME_name=NEWNAME
         # FreeBSD network script will rename the interface automatically.
-        return
-
-    def install_packages(self, pkglist):
-        self.update_package_sources()
-        self.package_command('install', pkgs=pkglist)
-
-    def package_command(self, command, args=None, pkgs=None):
-        if pkgs is None:
-            pkgs = []
+        pass
 
+    def _get_pkg_cmd_environ(self):
+        """Return environment vars used in *BSD package_command operations"""
         e = os.environ.copy()
         e['ASSUME_ALWAYS_YES'] = 'YES'
-
-        cmd = ['pkg']
-        if args and isinstance(args, str):
-            cmd.append(args)
-        elif args and isinstance(args, list):
-            cmd.extend(args)
-
-        if command:
-            cmd.append(command)
-
-        pkglist = util.expand_package_list('%s-%s', pkgs)
-        cmd.extend(pkglist)
-
-        # Allow the output of this to flow outwards (ie not be captured)
-        util.subp(cmd, env=e, capture=False)
-
-    def set_timezone(self, tz):
-        distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz))
+        return e
 
     def update_package_sources(self):
-        self._runner.run("update-sources", self.package_command,
-                         ["update"], freq=PER_INSTANCE)
+        self._runner.run(
+            "update-sources", self.package_command,
+            ["update"], freq=PER_INSTANCE)
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py
new file mode 100644
index 00000000..353eb671
--- /dev/null
+++ b/cloudinit/distros/netbsd.py
@@ -0,0 +1,133 @@
+# Copyright (C) 2019-2020 Gonéri Le Bouder
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import crypt
+import os
+import platform
+import six
+
+import cloudinit.distros.bsd
+from cloudinit import log as logging
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+class Distro(cloudinit.distros.bsd.BSD):
+    ci_sudoers_fn = '/usr/pkg/etc/sudoers.d/90-cloud-init-users'
+
+    group_add_cmd_prefix = ["groupadd"]
+    pkg_cmd_install_prefix = ["pkg_add", "-U"]
+    pkg_cmd_remove_prefix = ['pkg_delete']
+
+    def _get_add_member_to_group_cmd(self, member_name, group_name):
+        return ['usermod', '-G', group_name, member_name]
+
+    def add_user(self, name, **kwargs):
+        if util.is_user(name):
+            LOG.info("User %s already exists, skipping.", name)
+            return False
+
+        adduser_cmd = ['useradd']
+        log_adduser_cmd = ['useradd']
+
+        adduser_opts = {
+            "homedir": '-d',
+            "gecos": '-c',
+            "primary_group": '-g',
+            "groups": '-G',
+            "shell": '-s',
+        }
+        adduser_flags = {
+            "no_user_group": '--no-user-group',
+            "system": '--system',
+            "no_log_init": '--no-log-init',
+        }
+
+        for key, val in kwargs.items():
+            if (key in adduser_opts and val and
+               isinstance(val, six.string_types)):
+                adduser_cmd.extend([adduser_opts[key], val])
+
+            elif key in adduser_flags and val:
+                adduser_cmd.append(adduser_flags[key])
+                log_adduser_cmd.append(adduser_flags[key])
+
+        if 'no_create_home' not in kwargs or 'system' not in kwargs:
+            adduser_cmd += ['-m']
+            log_adduser_cmd += ['-m']
+
+        adduser_cmd += [name]
+        log_adduser_cmd += [name]
+
+        # Run the command
+        LOG.info("Adding user %s", name)
+        try:
+            util.subp(adduser_cmd, logstring=log_adduser_cmd)
+        except Exception:
+            util.logexc(LOG, "Failed to create user %s", name)
+            raise
+        # Set the password if it is provided
+        # For security consideration, only hashed passwd is assumed
+        passwd_val = kwargs.get('passwd', None)
+        if passwd_val is not None:
+            self.set_passwd(name, passwd_val, hashed=True)
+
+    def set_passwd(self, user, passwd, hashed=False):
+        if hashed:
+            hashed_pw = passwd
+        elif not hasattr(crypt, 'METHOD_BLOWFISH'):
+            # crypt.METHOD_BLOWFISH comes with Python 3.7 which is available
+            # on NetBSD 7 and 8.
+            LOG.error((
+                'Cannot set non-encrypted password for user %s. '
+                'Python >= 3.7 is required.'), user)
+            return
+        else:
+            method = crypt.METHOD_BLOWFISH  # pylint: disable=E1101
+            hashed_pw = crypt.crypt(
+                    passwd,
+                    crypt.mksalt(method))
+
+        try:
+            util.subp(['usermod', '-C', 'no', '-p', hashed_pw, user])
+        except Exception:
+            util.logexc(LOG, "Failed to set password for %s", user)
+            raise
+
+    def force_passwd_change(self, user):
+        try:
+            util.subp(['usermod', '-F', user])
+        except Exception:
+            util.logexc(LOG, "Failed to set pw expiration for %s", user)
+            raise
+
+    def lock_passwd(self, name):
+        try:
+            util.subp(['usermod', '-C', 'yes', name])
+        except Exception:
+            util.logexc(LOG, "Failed to lock user %s", name)
+            raise
+
+    def apply_locale(self, locale, out_fn=None):
+        LOG.debug('Cannot set the locale.')
+
+    def apply_network_config_names(self, netconfig):
+        LOG.debug('NetBSD cannot rename network interface.')
+
+    def _get_pkg_cmd_environ(self):
+        """Return environment vars used in *BSD package_command operations"""
+        os_release = platform.release()
+        os_arch = platform.machine()
+        e = os.environ.copy()
+        e['PKG_PATH'] = (
+                'http://cdn.netbsd.org/pub/pkgsrc/'
+                'packages/NetBSD/%s/%s/All') % (os_arch, os_release)
+        return e
+
+    def update_package_sources(self):
+        pass
+
+
+# vi: ts=4 expandtab
-- 
cgit v1.2.3


From 309e9a3cb33a16721d87650f9c1b50709e507356 Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Sat, 14 Mar 2020 08:31:33 -0400
Subject: util/netbsd: drop six usage (#252)

Drop remaining python six usage
---
 cloudinit/distros/netbsd.py |  4 +--
 cloudinit/util.py           | 61 +++++++++++++++++++++------------------------
 2 files changed, 29 insertions(+), 36 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py
index 353eb671..96794d76 100644
--- a/cloudinit/distros/netbsd.py
+++ b/cloudinit/distros/netbsd.py
@@ -5,7 +5,6 @@
 import crypt
 import os
 import platform
-import six
 
 import cloudinit.distros.bsd
 from cloudinit import log as logging
@@ -46,8 +45,7 @@ class Distro(cloudinit.distros.bsd.BSD):
         }
 
         for key, val in kwargs.items():
-            if (key in adduser_opts and val and
-               isinstance(val, six.string_types)):
+            if key in adduser_opts and val and isinstance(val, str):
                 adduser_cmd.extend([adduser_opts[key], val])
 
             elif key in adduser_flags and val:
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 718c6959..6c13f3f4 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -15,6 +15,7 @@ import glob
 import grp
 import gzip
 import hashlib
+import io
 import json
 import os
 import os.path
@@ -30,13 +31,11 @@ import string
 import subprocess
 import sys
 import time
+from urllib import parse
 
 from errno import ENOENT, ENOEXEC
 
 from base64 import b64decode, b64encode
-from six.moves.urllib import parse as urlparse
-
-import six
 
 from cloudinit import importer
 from cloudinit import log as logging
@@ -118,7 +117,7 @@ def target_path(target, path=None):
     # return 'path' inside target, accepting target as None
     if target in (None, ""):
         target = "/"
-    elif not isinstance(target, six.string_types):
+    elif not isinstance(target, str):
         raise ValueError("Unexpected input for target: %s" % target)
     else:
         target = os.path.abspath(target)
@@ -138,14 +137,14 @@ def target_path(target, path=None):
 
 def decode_binary(blob, encoding='utf-8'):
     # Converts a binary type into a text type using given encoding.
-    if isinstance(blob, six.string_types):
+    if isinstance(blob, str):
         return blob
     return blob.decode(encoding)
 
 
 def encode_text(text, encoding='utf-8'):
     # Converts a text string into a binary type using given encoding.
-    if isinstance(text, six.binary_type):
+    if isinstance(text, bytes):
         return text
     return text.encode(encoding)
 
@@ -175,8 +174,7 @@ def fully_decoded_payload(part):
     # bytes, first try to decode to str via CT charset, and failing that, try
     # utf-8 using surrogate escapes.
     cte_payload = part.get_payload(decode=True)
-    if (six.PY3 and
-            part.get_content_maintype() == 'text' and
+    if (part.get_content_maintype() == 'text' and
             isinstance(cte_payload, bytes)):
         charset = part.get_charset()
         if charset and charset.input_codec:
@@ -240,7 +238,7 @@ class ProcessExecutionError(IOError):
         else:
             self.description = description
 
-        if not isinstance(exit_code, six.integer_types):
+        if not isinstance(exit_code, int):
             self.exit_code = self.empty_attr
         else:
             self.exit_code = exit_code
@@ -281,7 +279,7 @@ class ProcessExecutionError(IOError):
         """
         if data is bytes object, decode
         """
-        return text.decode() if isinstance(text, six.binary_type) else text
+        return text.decode() if isinstance(text, bytes) else text
 
     def _indent_text(self, text, indent_level=8):
         """
@@ -290,7 +288,7 @@ class ProcessExecutionError(IOError):
         cr = '\n'
         indent = ' ' * indent_level
         # if input is bytes, return bytes
-        if isinstance(text, six.binary_type):
+        if isinstance(text, bytes):
             cr = cr.encode()
             indent = indent.encode()
         # remove any newlines at end of text first to prevent unneeded blank
@@ -322,9 +320,6 @@ class SeLinuxGuard(object):
             return
 
         path = os.path.realpath(self.path)
-        # path should be a string, not unicode
-        if six.PY2:
-            path = str(path)
         try:
             stats = os.lstat(path)
             self.selinux.matchpathcon(path, stats[stat.ST_MODE])
@@ -369,7 +364,7 @@ def is_true(val, addons=None):
     check_set = TRUE_STRINGS
     if addons:
         check_set = list(check_set) + addons
-    if six.text_type(val).lower().strip() in check_set:
+    if str(val).lower().strip() in check_set:
         return True
     return False
 
@@ -380,7 +375,7 @@ def is_false(val, addons=None):
     check_set = FALSE_STRINGS
     if addons:
         check_set = list(check_set) + addons
-    if six.text_type(val).lower().strip() in check_set:
+    if str(val).lower().strip() in check_set:
         return True
     return False
 
@@ -441,7 +436,7 @@ def uniq_merge_sorted(*lists):
 def uniq_merge(*lists):
     combined_list = []
     for a_list in lists:
-        if isinstance(a_list, six.string_types):
+        if isinstance(a_list, str):
             a_list = a_list.strip().split(",")
             # Kickout the empty ones
             a_list = [a for a in a_list if len(a)]
@@ -464,7 +459,7 @@ def clean_filename(fn):
 
 def decomp_gzip(data, quiet=True, decode=True):
     try:
-        buf = six.BytesIO(encode_text(data))
+        buf = io.BytesIO(encode_text(data))
         with contextlib.closing(gzip.GzipFile(None, "rb", 1, buf)) as gh:
             # E1101 is https://github.com/PyCQA/pylint/issues/1444
             if decode:
@@ -475,7 +470,7 @@ def decomp_gzip(data, quiet=True, decode=True):
         if quiet:
             return data
         else:
-            raise DecompressionError(six.text_type(e))
+            raise DecompressionError(str(e))
 
 
 def extract_usergroup(ug_pair):
@@ -567,7 +562,7 @@ def get_cfg_option_str(yobj, key, default=None):
     if key not in yobj:
         return default
     val = yobj[key]
-    if not isinstance(val, six.string_types):
+    if not isinstance(val, str):
         val = str(val)
     return val
 
@@ -707,7 +702,7 @@ def get_cfg_option_list(yobj, key, default=None):
     if isinstance(val, (list)):
         cval = [v for v in val]
         return cval
-    if not isinstance(val, six.string_types):
+    if not isinstance(val, str):
         val = str(val)
     return [val]
 
@@ -728,7 +723,7 @@ def get_cfg_by_path(yobj, keyp, default=None):
     @return: The value of the item at keyp."
         is not found."""
 
-    if isinstance(keyp, six.string_types):
+    if isinstance(keyp, str):
         keyp = keyp.split("/")
     cur = yobj
     for tok in keyp:
@@ -826,7 +821,7 @@ def make_url(scheme, host, port=None,
     pieces.append(query or '')
     pieces.append(fragment or '')
 
-    return urlparse.urlunparse(pieces)
+    return parse.urlunparse(pieces)
 
 
 def mergemanydict(srcs, reverse=False):
@@ -1035,7 +1030,7 @@ def read_conf_with_confd(cfgfile):
     if "conf_d" in cfg:
         confd = cfg['conf_d']
         if confd:
-            if not isinstance(confd, six.string_types):
+            if not isinstance(confd, str):
                 raise TypeError(("Config file %s contains 'conf_d' "
                                  "with non-string type %s") %
                                 (cfgfile, type_utils.obj_name(confd)))
@@ -1227,7 +1222,7 @@ def is_resolvable_url(url):
     """determine if this url is resolvable (existing or ip)."""
     return log_time(logfunc=LOG.debug, msg="Resolving URL: " + url,
                     func=is_resolvable,
-                    args=(urlparse.urlparse(url).hostname,))
+                    args=(parse.urlparse(url).hostname,))
 
 
 def search_for_mirror(candidates):
@@ -1378,7 +1373,7 @@ def uniq_list(in_list):
 
 def load_file(fname, read_cb=None, quiet=False, decode=True):
     LOG.debug("Reading from %s (quiet=%s)", fname, quiet)
-    ofh = six.BytesIO()
+    ofh = io.BytesIO()
     try:
         with open(fname, 'rb') as ifh:
             pipe_in_out(ifh, ofh, chunk_cb=read_cb)
@@ -2074,13 +2069,13 @@ def subp(args, data=None, rcs=None, env=None, capture=True,
     # Popen converts entries in the arguments array from non-bytes to bytes.
     # When locale is unset it may use ascii for that encoding which can
     # cause UnicodeDecodeErrors. (LP: #1751051)
-    if isinstance(args, six.binary_type):
+    if isinstance(args, bytes):
         bytes_args = args
-    elif isinstance(args, six.string_types):
+    elif isinstance(args, str):
         bytes_args = args.encode("utf-8")
     else:
         bytes_args = [
-            x if isinstance(x, six.binary_type) else x.encode("utf-8")
+            x if isinstance(x, bytes) else x.encode("utf-8")
             for x in args]
     try:
         sp = subprocess.Popen(bytes_args, stdout=stdout,
@@ -2159,10 +2154,10 @@ def shellify(cmdlist, add_header=True):
         if isinstance(args, (list, tuple)):
             fixed = []
             for f in args:
-                fixed.append("'%s'" % (six.text_type(f).replace("'", escaped)))
+                fixed.append("'%s'" % (str(f).replace("'", escaped)))
             content = "%s%s\n" % (content, ' '.join(fixed))
             cmds_made += 1
-        elif isinstance(args, six.string_types):
+        elif isinstance(args, str):
             content = "%s%s\n" % (content, args)
             cmds_made += 1
         else:
@@ -2288,7 +2283,7 @@ def expand_package_list(version_fmt, pkgs):
 
     pkglist = []
     for pkg in pkgs:
-        if isinstance(pkg, six.string_types):
+        if isinstance(pkg, str):
             pkglist.append(pkg)
             continue
 
@@ -2788,7 +2783,7 @@ def read_dmi_data(key):
 
 def message_from_string(string):
     if sys.version_info[:2] < (2, 7):
-        return email.message_from_file(six.StringIO(string))
+        return email.message_from_file(io.StringIO(string))
     return email.message_from_string(string)
 
 
-- 
cgit v1.2.3


From 53b6d1d6cdd1691b4379df4829109db40097b61b Mon Sep 17 00:00:00 2001
From: Gonéri Le Bouder <goneri@lebouder.net>
Date: Tue, 24 Mar 2020 16:35:52 -0400
Subject: freebsd: ensure package update works (#273)

Currently, `cc_package_update_upgrade_install.py` fails because
`package_command()` does not know how to do an update on FreeBSD.

```
2020-03-23 20:01:53,995 - util.py[DEBUG]: Package update failed
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/site-packages/cloud_init-20.1-py3.7.egg/cloudinit/config/cc_package_update_upgrade_install.py", line 85, in handle
    cloud.distro.update_package_sources()
  File "/usr/local/lib/python3.7/site-packages/cloud_init-20.1-py3.7.egg/cloudinit/distros/freebsd.py", line 158, in update_package_sources
    ["update"], freq=PER_INSTANCE)
  File "/usr/local/lib/python3.7/site-packages/cloud_init-20.1-py3.7.egg/cloudinit/helpers.py", line 185, in run
    results = functor(*args)
  File "/usr/local/lib/python3.7/site-packages/cloud_init-20.1-py3.7.egg/cloudinit/distros/bsd.py", line 102, in package_command
    cmd.extend(pkglist)
UnboundLocalError: local variable 'cmd' referenced before assignment
```

This commit defines a new `pkg_cmd_update_prefix` key. If it's empty, we
don't do any update, otherwise we use the value to update the package
manager.
---
 cloudinit/distros/bsd.py     | 9 +++++++--
 cloudinit/distros/freebsd.py | 1 +
 2 files changed, 8 insertions(+), 2 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py
index e9b84edc..c58f897c 100644
--- a/cloudinit/distros/bsd.py
+++ b/cloudinit/distros/bsd.py
@@ -18,6 +18,9 @@ class BSD(distros.Distro):
     group_add_cmd_prefix = []
     pkg_cmd_install_prefix = []
     pkg_cmd_remove_prefix = []
+    # There is no need to update the package cache on NetBSD and OpenBSD
+    # TODO neither freebsd nor netbsd handles a command 'upgrade'
+    pkg_cmd_update_prefix = None
 
     def __init__(self, name, cfg, paths):
         super().__init__(name, cfg, paths)
@@ -86,12 +89,14 @@ class BSD(distros.Distro):
         if pkgs is None:
             pkgs = []
 
-        # TODO neither freebsd nor netbsd handles a command 'upgrade'
-        # provided by cloudinit/config/cc_package_update_upgrade_install.py
         if command == 'install':
             cmd = self.pkg_cmd_install_prefix
         elif command == 'remove':
             cmd = self.pkg_cmd_remove_prefix
+        elif command == 'update':
+            if not self.pkg_cmd_update_prefix:
+                return
+            cmd = self.pkg_cmd_update_prefix
 
         if args and isinstance(args, str):
             cmd.append(args)
diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py
index a775ae51..71a6195b 100644
--- a/cloudinit/distros/freebsd.py
+++ b/cloudinit/distros/freebsd.py
@@ -24,6 +24,7 @@ class Distro(cloudinit.distros.bsd.BSD):
     group_add_cmd_prefix = ['pw', 'group', 'add']
     pkg_cmd_install_prefix = ["pkg", "install"]
     pkg_cmd_remove_prefix = ["pkg", "remove"]
+    pkg_cmd_update_prefix = ["pkg", "update"]
 
     def _select_hostname(self, hostname, fqdn):
         # Should be FQDN if available. See rc.conf(5) in FreeBSD
-- 
cgit v1.2.3


From c5e949c02a1d5226ae9b1cb39846db19d223c6c2 Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Wed, 25 Mar 2020 09:44:16 -0400
Subject: distros/tests/test_init: add tests for _get_package_mirror_info
 (#272)

---
 cloudinit/distros/tests/__init__.py  |  0
 cloudinit/distros/tests/test_init.py | 83 ++++++++++++++++++++++++++++++++++++
 2 files changed, 83 insertions(+)
 create mode 100644 cloudinit/distros/tests/__init__.py
 create mode 100644 cloudinit/distros/tests/test_init.py

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/tests/__init__.py b/cloudinit/distros/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/cloudinit/distros/tests/test_init.py b/cloudinit/distros/tests/test_init.py
new file mode 100644
index 00000000..707a0a49
--- /dev/null
+++ b/cloudinit/distros/tests/test_init.py
@@ -0,0 +1,83 @@
+# Copyright (C) 2020 Canonical Ltd.
+#
+# Author: Daniel Watkins <oddbloke@ubuntu.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+"""Tests for cloudinit/distros/__init__.py"""
+
+from unittest import mock
+
+import pytest
+
+from cloudinit.distros import _get_package_mirror_info
+
+
+class TestGetPackageMirrorInfo:
+    """
+    Tests for cloudinit.distros._get_package_mirror_info.
+
+    These supplement the tests in tests/unittests/test_distros/test_generic.py
+    which are more focused on testing a single production-like configuration.
+    These tests are more focused on specific aspects of the unit under test.
+    """
+
+    @pytest.mark.parametrize('mirror_info,expected', [
+        # Empty info gives empty return
+        ({}, {}),
+        # failsafe values used if present
+        ({'failsafe': {'primary': 'value', 'security': 'other'}},
+         {'primary': 'value', 'security': 'other'}),
+        # search values used if present
+        ({'search': {'primary': ['value'], 'security': ['other']}},
+         {'primary': ['value'], 'security': ['other']}),
+        # failsafe values used if search value not present
+        ({'search': {'primary': ['value']}, 'failsafe': {'security': 'other'}},
+         {'primary': ['value'], 'security': 'other'})
+    ])
+    def test_get_package_mirror_info_failsafe(self, mirror_info, expected):
+        """
+        Test the interaction between search and failsafe inputs
+
+        (This doesn't test the case where the mirror_filter removes all search
+        options; test_failsafe_used_if_all_search_results_filtered_out covers
+        that.)
+        """
+        assert expected == _get_package_mirror_info(mirror_info,
+                                                    mirror_filter=lambda x: x)
+
+    def test_failsafe_used_if_all_search_results_filtered_out(self):
+        """Test the failsafe option used if all search options eliminated."""
+        mirror_info = {
+            'search': {'primary': ['value']}, 'failsafe': {'primary': 'other'}
+        }
+        assert {'primary': 'other'} == _get_package_mirror_info(
+            mirror_info, mirror_filter=lambda x: False)
+
+    @pytest.mark.parametrize('availability_zone,region,patterns,expected', (
+        # Test ec2_region alone
+        ('fk-fake-1f', None, ['EC2-%(ec2_region)s'], ['EC2-fk-fake-1']),
+        # Test availability_zone alone
+        ('fk-fake-1f', None, ['AZ-%(availability_zone)s'], ['AZ-fk-fake-1f']),
+        # Test region alone
+        (None, 'fk-fake-1', ['RG-%(region)s'], ['RG-fk-fake-1']),
+        # Test that ec2_region is not available for non-matching AZs
+        ('fake-fake-1f', None,
+         ['EC2-%(ec2_region)s', 'AZ-%(availability_zone)s'],
+         ['AZ-fake-fake-1f']),
+        # Test that template order maintained
+        (None, 'fake-region', ['RG-%(region)s-2', 'RG-%(region)s-1'],
+         ['RG-fake-region-2', 'RG-fake-region-1']),
+    ))
+    def test_substitution(self, availability_zone, region, patterns, expected):
+        """Test substitution works as expected."""
+        m_data_source = mock.Mock(
+            availability_zone=availability_zone, region=region
+        )
+        mirror_info = {'search': {'primary': patterns}}
+
+        ret = _get_package_mirror_info(
+            mirror_info,
+            data_source=m_data_source,
+            mirror_filter=lambda x: x
+        )
+        assert {'primary': expected} == ret
-- 
cgit v1.2.3


From 44039629e539ed48298703028ac8f10ad3c60d6e Mon Sep 17 00:00:00 2001
From: Gonéri Le Bouder <goneri@lebouder.net>
Date: Thu, 26 Mar 2020 16:07:51 -0400
Subject: add Openbsd support (#147)

- tested on OpenBSD 6.6
- tested on OpenStack without config drive, and NoCloud with ISO config
  drive
---
 cloudinit/distros/bsd.py                   |  6 ++++
 cloudinit/distros/netbsd.py                | 13 +++++++--
 cloudinit/distros/openbsd.py               | 47 ++++++++++++++++++++++++++++++
 cloudinit/net/__init__.py                  | 23 +++++++++++++--
 cloudinit/net/openbsd.py                   | 44 ++++++++++++++++++++++++++++
 cloudinit/net/renderers.py                 |  5 +++-
 cloudinit/sources/DataSourceConfigDrive.py |  2 +-
 cloudinit/util.py                          | 31 +++++++++++++++++++-
 config/cloud.cfg.tmpl                      | 16 +++++++---
 doc/rtd/topics/network-config.rst          |  2 +-
 setup.py                                   |  5 ++--
 tests/unittests/test_util.py               | 19 ++++++++++++
 tools/build-on-openbsd                     | 27 +++++++++++++++++
 tools/render-cloudcfg                      |  5 ++--
 14 files changed, 228 insertions(+), 17 deletions(-)
 create mode 100644 cloudinit/distros/openbsd.py
 create mode 100644 cloudinit/net/openbsd.py
 create mode 100755 tools/build-on-openbsd

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py
index c58f897c..efc73d5d 100644
--- a/cloudinit/distros/bsd.py
+++ b/cloudinit/distros/bsd.py
@@ -114,3 +114,9 @@ class BSD(distros.Distro):
 
     def set_timezone(self, tz):
         distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz))
+
+    def apply_locale(self, locale, out_fn=None):
+        LOG.debug('Cannot set the locale.')
+
+    def apply_network_config_names(self, netconfig):
+        LOG.debug('Cannot rename network interface.')
diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py
index 96794d76..69d07846 100644
--- a/cloudinit/distros/netbsd.py
+++ b/cloudinit/distros/netbsd.py
@@ -13,7 +13,13 @@ from cloudinit import util
 LOG = logging.getLogger(__name__)
 
 
-class Distro(cloudinit.distros.bsd.BSD):
+class NetBSD(cloudinit.distros.bsd.BSD):
+    """
+    Distro subclass for NetBSD.
+
+    (N.B. OpenBSD inherits from this class.)
+    """
+
     ci_sudoers_fn = '/usr/pkg/etc/sudoers.d/90-cloud-init-users'
 
     group_add_cmd_prefix = ["groupadd"]
@@ -115,7 +121,7 @@ class Distro(cloudinit.distros.bsd.BSD):
         LOG.debug('NetBSD cannot rename network interface.')
 
     def _get_pkg_cmd_environ(self):
-        """Return environment vars used in *BSD package_command operations"""
+        """Return env vars used in NetBSD package_command operations"""
         os_release = platform.release()
         os_arch = platform.machine()
         e = os.environ.copy()
@@ -128,4 +134,7 @@ class Distro(cloudinit.distros.bsd.BSD):
         pass
 
 
+class Distro(NetBSD):
+    pass
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/distros/openbsd.py b/cloudinit/distros/openbsd.py
new file mode 100644
index 00000000..dbe1f069
--- /dev/null
+++ b/cloudinit/distros/openbsd.py
@@ -0,0 +1,47 @@
+# Copyright (C) 2019-2020 Gonéri Le Bouder
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import os
+import platform
+
+import cloudinit.distros.netbsd
+from cloudinit import log as logging
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+class Distro(cloudinit.distros.netbsd.NetBSD):
+    hostname_conf_fn = '/etc/myname'
+
+    def _read_hostname(self, filename, default=None):
+        return util.load_file(self.hostname_conf_fn)
+
+    def _write_hostname(self, hostname, filename):
+        content = hostname + '\n'
+        util.write_file(self.hostname_conf_fn, content)
+
+    def _get_add_member_to_group_cmd(self, member_name, group_name):
+        return ['usermod', '-G', group_name, member_name]
+
+    def lock_passwd(self, name):
+        try:
+            util.subp(['usermod', '-p', '*', name])
+        except Exception:
+            util.logexc(LOG, "Failed to lock user %s", name)
+            raise
+
+    def _get_pkg_cmd_environ(self):
+        """Return env vars used in OpenBSD package_command operations"""
+        os_release = platform.release()
+        os_arch = platform.machine()
+        e = os.environ.copy()
+        e['PKG_PATH'] = (
+                'ftp://ftp.openbsd.org/pub/OpenBSD/{os_release}/'
+                'packages/{os_arch}/').format(
+                        os_arch=os_arch, os_release=os_release)
+        return e
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 400d7870..67e3d578 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -334,13 +334,13 @@ def find_fallback_nic(blacklist_drivers=None):
     """Return the name of the 'fallback' network device."""
     if util.is_FreeBSD():
         return find_fallback_nic_on_freebsd(blacklist_drivers)
-    elif util.is_NetBSD():
-        return find_fallback_nic_on_netbsd(blacklist_drivers)
+    elif util.is_NetBSD() or util.is_OpenBSD():
+        return find_fallback_nic_on_netbsd_or_openbsd(blacklist_drivers)
     else:
         return find_fallback_nic_on_linux(blacklist_drivers)
 
 
-def find_fallback_nic_on_netbsd(blacklist_drivers=None):
+def find_fallback_nic_on_netbsd_or_openbsd(blacklist_drivers=None):
     values = list(sorted(
         get_interfaces_by_mac().values(),
         key=natural_sort_key))
@@ -811,6 +811,8 @@ def get_interfaces_by_mac():
         return get_interfaces_by_mac_on_freebsd()
     elif util.is_NetBSD():
         return get_interfaces_by_mac_on_netbsd()
+    elif util.is_OpenBSD():
+        return get_interfaces_by_mac_on_openbsd()
     else:
         return get_interfaces_by_mac_on_linux()
 
@@ -857,6 +859,21 @@ def get_interfaces_by_mac_on_netbsd():
     return ret
 
 
+def get_interfaces_by_mac_on_openbsd():
+    ret = {}
+    re_field_match = (
+        r"(?P<ifname>\w+).*lladdr\s"
+        r"(?P<mac>([\da-f]{2}[:-]){5}([\da-f]{2})).*")
+    (out, _) = util.subp(['ifconfig', '-a'])
+    if_lines = re.sub(r'\n\s+', ' ', out).splitlines()
+    for line in if_lines:
+        m = re.match(re_field_match, line)
+        if m:
+            fields = m.groupdict()
+            ret[fields['mac']] = fields['ifname']
+    return ret
+
+
 def get_interfaces_by_mac_on_linux():
     """Build a dictionary of tuples {mac: name}.
 
diff --git a/cloudinit/net/openbsd.py b/cloudinit/net/openbsd.py
new file mode 100644
index 00000000..b9897e90
--- /dev/null
+++ b/cloudinit/net/openbsd.py
@@ -0,0 +1,44 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit import log as logging
+from cloudinit import util
+import cloudinit.net.bsd
+
+LOG = logging.getLogger(__name__)
+
+
+class Renderer(cloudinit.net.bsd.BSDRenderer):
+
+    def write_config(self):
+        for device_name, v in self.interface_configurations.items():
+            if_file = 'etc/hostname.{}'.format(device_name)
+            fn = util.target_path(self.target, if_file)
+            if device_name in self.dhcp_interfaces():
+                content = 'dhcp\n'
+            elif isinstance(v, dict):
+                try:
+                    content = "inet {address} {netmask}\n".format(
+                            address=v['address'],
+                            netmask=v['netmask'])
+                except KeyError:
+                    LOG.error(
+                        "Invalid static configuration for %s",
+                        device_name)
+            util.write_file(fn, content)
+
+    def start_services(self, run=False):
+        if not self._postcmds:
+            LOG.debug("openbsd generate postcmd disabled")
+            return
+        util.subp(['sh', '/etc/netstart'], capture=True)
+
+    def set_route(self, network, netmask, gateway):
+        if network == '0.0.0.0':
+            if_file = 'etc/mygate'
+            fn = util.target_path(self.target, if_file)
+            content = gateway + '\n'
+            util.write_file(fn, content)
+
+
+def available(target=None):
+    return util.is_OpenBSD()
diff --git a/cloudinit/net/renderers.py b/cloudinit/net/renderers.py
index e4bcae9d..e2de4d55 100644
--- a/cloudinit/net/renderers.py
+++ b/cloudinit/net/renderers.py
@@ -5,6 +5,7 @@ from . import freebsd
 from . import netbsd
 from . import netplan
 from . import RendererNotFoundError
+from . import openbsd
 from . import sysconfig
 
 NAME_TO_RENDERER = {
@@ -12,10 +13,12 @@ NAME_TO_RENDERER = {
     "freebsd": freebsd,
     "netbsd": netbsd,
     "netplan": netplan,
+    "openbsd": openbsd,
     "sysconfig": sysconfig,
 }
 
-DEFAULT_PRIORITY = ["eni", "sysconfig", "netplan", "freebsd", "netbsd"]
+DEFAULT_PRIORITY = ["eni", "sysconfig", "netplan", "freebsd",
+                    "netbsd", "openbsd"]
 
 
 def search(priority=None, target=None, first=False):
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index dee8cde4..ae31934b 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -72,7 +72,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
             dslist = self.sys_cfg.get('datasource_list')
             for dev in find_candidate_devs(dslist=dslist):
                 mtype = None
-                if (util.is_FreeBSD() or util.is_NetBSD()):
+                if util.is_BSD():
                     if dev.startswith("/dev/cd"):
                         mtype = "cd9660"
                 try:
diff --git a/cloudinit/util.py b/cloudinit/util.py
index db60b9d2..541a486b 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -557,6 +557,11 @@ def is_NetBSD():
     return system_info()['variant'] == "netbsd"
 
 
+@lru_cache()
+def is_OpenBSD():
+    return system_info()['variant'] == "openbsd"
+
+
 def get_cfg_option_bool(yobj, key, default=False):
     if key not in yobj:
         return default
@@ -679,7 +684,7 @@ def system_info():
             var = 'suse'
         else:
             var = 'linux'
-    elif system in ('windows', 'darwin', "freebsd", "netbsd"):
+    elif system in ('windows', 'darwin', "freebsd", "netbsd", "openbsd"):
         var = system
 
     info['variant'] = var
@@ -1279,6 +1284,27 @@ def find_devs_with_netbsd(criteria=None, oformat='device',
     return []
 
 
+def find_devs_with_openbsd(criteria=None, oformat='device',
+                           tag=None, no_cache=False, path=None):
+    out, _err = subp(['sysctl', '-n', 'hw.disknames'], rcs=[0])
+    devlist = []
+    for entry in out.split(','):
+        if not entry.endswith(':'):
+            # ffs partition with a serial, not a config-drive
+            continue
+        if entry == 'fd0:':
+            continue
+        part_id = 'a' if entry.startswith('cd') else 'i'
+        devlist.append(entry[:-1] + part_id)
+    if criteria == "TYPE=iso9660":
+        devlist = [i for i in devlist if i.startswith('cd')]
+    elif criteria in ["LABEL=CONFIG-2", "TYPE=vfat"]:
+        devlist = [i for i in devlist if not i.startswith('cd')]
+    elif criteria:
+        LOG.debug("Unexpected criteria: %s", criteria)
+    return ['/dev/' + i for i in devlist]
+
+
 def find_devs_with(criteria=None, oformat='device',
                    tag=None, no_cache=False, path=None):
     """
@@ -1291,6 +1317,9 @@ def find_devs_with(criteria=None, oformat='device',
     if is_NetBSD():
         return find_devs_with_netbsd(criteria, oformat,
                                      tag, no_cache, path)
+    elif is_OpenBSD():
+        return find_devs_with_openbsd(criteria, oformat,
+                                      tag, no_cache, path)
 
     blk_id_cmd = ['blkid']
     options = []
diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl
index 50cfbb2f..2b052c0f 100644
--- a/config/cloud.cfg.tmpl
+++ b/config/cloud.cfg.tmpl
@@ -2,7 +2,7 @@
 # The top level settings are used as module
 # and system configuration.
 
-{% if variant in ["freebsd", "netbsd"] %}
+{% if variant.endswith("bsd") %}
 syslog_fix_perms: root:wheel
 {% elif variant in ["suse"] %}
 syslog_fix_perms: root:root
@@ -33,7 +33,7 @@ ssh_pwauth:   0
 # This will cause the set+update hostname module to not operate (if true)
 preserve_hostname: false
 
-{% if variant in ["freebsd", "netbsd"] %}
+{% if variant.endswith("bsd") %}
 # This should not be required, but leave it in place until the real cause of
 # not finding -any- datasources is resolved.
 datasource_list: ['NoCloud', 'ConfigDrive', 'Azure', 'OpenStack', 'Ec2']
@@ -147,7 +147,7 @@ cloud_final_modules:
 # (not accessible to handlers/transforms)
 system_info:
    # This will affect which distro class gets used
-{% if variant in ["amazon", "arch", "centos", "debian", "fedora", "freebsd", "netbsd", "rhel", "suse", "ubuntu"] %}
+{% if variant in ["amazon", "arch", "centos", "debian", "fedora", "freebsd", "netbsd", "openbsd", "rhel", "suse", "ubuntu"] %}
    distro: {{ variant }}
 {% else %}
    # Unknown/fallback distro.
@@ -238,8 +238,16 @@ system_info:
      groups: [wheel]
      sudo: ["ALL=(ALL) NOPASSWD:ALL"]
      shell: /bin/sh
+{% elif variant in ["openbsd"] %}
+   default_user:
+     name: openbsd
+     lock_passwd: True
+     gecos: OpenBSD
+     groups: [wheel]
+     sudo: ["ALL=(ALL) NOPASSWD:ALL"]
+     shell: /bin/ksh
 {% endif %}
-{% if variant in ["freebsd", "netbsd"] %}
+{% if variant in ["freebsd", "netbsd", "openbsd"] %}
    network:
       renderers: ['{{ variant }}']
 {% endif %}
diff --git a/doc/rtd/topics/network-config.rst b/doc/rtd/topics/network-config.rst
index ba00a889..8eeadebf 100644
--- a/doc/rtd/topics/network-config.rst
+++ b/doc/rtd/topics/network-config.rst
@@ -197,7 +197,7 @@ supplying an updated configuration in cloud-config. ::
 
   system_info:
     network:
-      renderers: ['netplan', 'eni', 'sysconfig', 'freebsd', 'netbsd']
+      renderers: ['netplan', 'eni', 'sysconfig', 'freebsd', 'netbsd', 'openbsd']
 
 
 Network Configuration Tools
diff --git a/setup.py b/setup.py
index 69bb1a51..22b32f6f 100755
--- a/setup.py
+++ b/setup.py
@@ -15,6 +15,7 @@ import os
 import shutil
 import sys
 import tempfile
+import platform
 
 import setuptools
 from setuptools.command.install import install
@@ -230,7 +231,7 @@ class InitsysInstallData(install):
         if self.init_system and isinstance(self.init_system, str):
             self.init_system = self.init_system.split(",")
 
-        if len(self.init_system) == 0:
+        if len(self.init_system) == 0 and not platform.system().endswith('BSD'):
             self.init_system = ['systemd']
 
         bad = [f for f in self.init_system if f not in INITSYS_TYPES]
@@ -274,7 +275,7 @@ data_files = [
     (USR + '/share/doc/cloud-init/examples/seed',
         [f for f in glob('doc/examples/seed/*') if is_f(f)]),
 ]
-if os.uname()[0] not in ['FreeBSD', 'NetBSD']:
+if not platform.system().endswith('BSD'):
     data_files.extend([
         (ETC + '/NetworkManager/dispatcher.d/',
          ['tools/hook-network-manager']),
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 5b2eaa69..2900997b 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -1171,4 +1171,23 @@ class TestGetProcEnv(helpers.TestCase):
         my_ppid = os.getppid()
         self.assertEqual(my_ppid, util.get_proc_ppid(my_pid))
 
+
+@mock.patch('cloudinit.util.subp')
+def test_find_devs_with_openbsd(m_subp):
+    m_subp.return_value = (
+        'cd0:,sd0:630d98d32b5d3759,sd1:,fd0:', ''
+    )
+    devlist = util.find_devs_with_openbsd()
+    assert devlist == ['/dev/cd0a', '/dev/sd1i']
+
+
+@mock.patch('cloudinit.util.subp')
+def test_find_devs_with_openbsd_with_criteria(m_subp):
+    m_subp.return_value = (
+        'cd0:,sd0:630d98d32b5d3759,sd1:,fd0:', ''
+    )
+    devlist = util.find_devs_with_openbsd(criteria="TYPE=iso9660")
+    assert devlist == ['/dev/cd0a']
+
+
 # vi: ts=4 expandtab
diff --git a/tools/build-on-openbsd b/tools/build-on-openbsd
new file mode 100755
index 00000000..ca028606
--- /dev/null
+++ b/tools/build-on-openbsd
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+fail() { echo "FAILED:" "$@" 1>&2; exit 1; }
+
+# Check dependencies:
+depschecked=/tmp/c-i.dependencieschecked
+pkgs="
+   bash
+   dmidecode
+   py3-configobj
+   py3-jinja2
+   py3-jsonschema
+   py3-oauthlib
+   py3-requests
+   py3-setuptools
+   py3-six
+   py3-yaml
+   sudo--
+"
+[ -f "$depschecked" ] || pkg_add ${pkgs} || fail "install packages"
+
+touch $depschecked
+
+python3 setup.py build
+python3 setup.py install -O1 --distro openbsd --skip-build
+
+echo "Installation completed."
diff --git a/tools/render-cloudcfg b/tools/render-cloudcfg
index 50f85369..9322b2c3 100755
--- a/tools/render-cloudcfg
+++ b/tools/render-cloudcfg
@@ -4,8 +4,9 @@ import argparse
 import os
 import sys
 
-VARIANTS = ["amazon", "arch", "centos", "debian", "fedora", "freebsd", "netbsd",
-            "rhel", "suse", "ubuntu", "unknown"]
+VARIANTS = ["amazon", "arch", "centos", "debian", "fedora", "freebsd",
+            "netbsd", "openbsd", "rhel", "suse", "ubuntu", "unknown"]
+
 
 if "avoid-pep8-E402-import-not-top-of-file":
     _tdir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
-- 
cgit v1.2.3


From c478d0bff412c67280dfe8f08568de733f9425a1 Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Tue, 31 Mar 2020 13:52:21 -0400
Subject: distros: replace invalid characters in mirror URLs with hyphens
 (#291)

This modifies _get_package_mirror_info to convert the hostnames of generated mirror URLs to their IDNA form, and then iterate through them replacing any invalid characters (i.e. anything other than letters, digits or a hyphen) with a hyphen.

This commit introduces the following changes in behaviour:

* generated mirror URLs with Unicode characters in their hostnames will have their hostnames converted to their all-ASCII IDNA form
* generated mirror URLs with invalid-for-hostname characters in their hostname will have those characters converted to hyphens
* generated mirror URLs which cannot be parsed by `urllib.parse.urlsplit` will not be considered for use
  * other configured patterns will still be considered
  * if all configured patterns fail to produce a URL that parses then the fallback mirror URL will be used

LP: #1868232
---
 cloudinit/distros/__init__.py        | 109 ++++++++++++++++++++++++++++++++++-
 cloudinit/distros/tests/test_init.py |  84 +++++++++++++++++++++------
 2 files changed, 174 insertions(+), 19 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 92598a2d..86a22c3e 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -13,6 +13,8 @@ import abc
 import os
 import re
 import stat
+import string
+import urllib.parse
 from io import StringIO
 
 from cloudinit import importer
@@ -50,6 +52,9 @@ _EC2_AZ_RE = re.compile('^[a-z][a-z]-(?:[a-z]+-)+[0-9][a-z]$')
 # Default NTP Client Configurations
 PREFERRED_NTP_CLIENTS = ['chrony', 'systemd-timesyncd', 'ntp', 'ntpdate']
 
+# Letters/Digits/Hyphen characters, for use in domain name validation
+LDH_ASCII_CHARS = string.ascii_letters + string.digits + "-"
+
 
 class Distro(metaclass=abc.ABCMeta):
 
@@ -720,6 +725,102 @@ class Distro(metaclass=abc.ABCMeta):
                 LOG.info("Added user '%s' to group '%s'", member, name)
 
 
+def _apply_hostname_transformations_to_url(url: str, transformations: list):
+    """
+    Apply transformations to a URL's hostname, return transformed URL.
+
+    This is a separate function because unwrapping and rewrapping only the
+    hostname portion of a URL is complex.
+
+    :param url:
+        The URL to operate on.
+    :param transformations:
+        A list of ``(str) -> Optional[str]`` functions, which will be applied
+        in order to the hostname portion of the URL.  If any function
+        (regardless of ordering) returns None, ``url`` will be returned without
+        any modification.
+
+    :return:
+        A string whose value is ``url`` with the hostname ``transformations``
+        applied, or ``None`` if ``url`` is unparseable.
+    """
+    try:
+        parts = urllib.parse.urlsplit(url)
+    except ValueError:
+        # If we can't even parse the URL, we shouldn't use it for anything
+        return None
+    new_hostname = parts.hostname
+
+    for transformation in transformations:
+        new_hostname = transformation(new_hostname)
+        if new_hostname is None:
+            # If a transformation returns None, that indicates we should abort
+            # processing and return `url` unmodified
+            return url
+
+    new_netloc = new_hostname
+    if parts.port is not None:
+        new_netloc = "{}:{}".format(new_netloc, parts.port)
+    return urllib.parse.urlunsplit(parts._replace(netloc=new_netloc))
+
+
+def _sanitize_mirror_url(url: str):
+    """
+    Given a mirror URL, replace or remove any invalid URI characters.
+
+    This performs the following actions on the URL's hostname:
+      * Checks if it is an IP address, returning the URL immediately if it is
+      * Converts it to its IDN form (see below for details)
+      * Replaces any non-Letters/Digits/Hyphen (LDH) characters in it with
+        hyphens
+      * TODO: Remove any leading/trailing hyphens from each domain name label
+
+    Before we replace any invalid domain name characters, we first need to
+    ensure that any valid non-ASCII characters in the hostname will not be
+    replaced, by ensuring the hostname is in its Internationalized domain name
+    (IDN) representation (see RFC 5890).  This conversion has to be applied to
+    the whole hostname (rather than just the substitution variables), because
+    the Punycode algorithm used by IDNA transcodes each part of the hostname as
+    a whole string (rather than encoding individual characters).  It cannot be
+    applied to the whole URL, because (a) the Punycode algorithm expects to
+    operate on domain names so doesn't output a valid URL, and (b) non-ASCII
+    characters in non-hostname parts of the URL aren't encoded via Punycode.
+
+    To put this in RFC 5890's terminology: before we remove or replace any
+    characters from our domain name (which we do to ensure that each label is a
+    valid LDH Label), we first ensure each label is in its A-label form.
+
+    (Note that Python's builtin idna encoding is actually IDNA2003, not
+    IDNA2008.  This changes the specifics of how some characters are encoded to
+    ASCII, but doesn't affect the logic here.)
+
+    :param url:
+        The URL to operate on.
+
+    :return:
+        A sanitized version of the URL, which will have been IDNA encoded if
+        necessary, or ``None`` if the generated string is not a parseable URL.
+    """
+    # Acceptable characters are LDH characters, plus "." to separate each label
+    acceptable_chars = LDH_ASCII_CHARS + "."
+    transformations = [
+        # This is an IP address, not a hostname, so no need to apply the
+        # transformations
+        lambda hostname: None if net.is_ip_address(hostname) else hostname,
+
+        # Encode with IDNA to get the correct characters (as `bytes`), then
+        # decode with ASCII so we return a `str`
+        lambda hostname: hostname.encode('idna').decode('ascii'),
+
+        # Replace any unacceptable characters with "-"
+        lambda hostname: ''.join(
+            c if c in acceptable_chars else "-" for c in hostname
+        ),
+    ]
+
+    return _apply_hostname_transformations_to_url(url, transformations)
+
+
 def _get_package_mirror_info(mirror_info, data_source=None,
                              mirror_filter=util.search_for_mirror):
     # given a arch specific 'mirror_info' entry (from package_mirrors)
@@ -748,9 +849,13 @@ def _get_package_mirror_info(mirror_info, data_source=None,
         mirrors = []
         for tmpl in searchlist:
             try:
-                mirrors.append(tmpl % subst)
+                mirror = tmpl % subst
             except KeyError:
-                pass
+                continue
+
+            mirror = _sanitize_mirror_url(mirror)
+            if mirror is not None:
+                mirrors.append(mirror)
 
         found = mirror_filter(mirrors)
         if found:
diff --git a/cloudinit/distros/tests/test_init.py b/cloudinit/distros/tests/test_init.py
index 707a0a49..daa81ab8 100644
--- a/cloudinit/distros/tests/test_init.py
+++ b/cloudinit/distros/tests/test_init.py
@@ -9,7 +9,18 @@ from unittest import mock
 
 import pytest
 
-from cloudinit.distros import _get_package_mirror_info
+from cloudinit.distros import _get_package_mirror_info, LDH_ASCII_CHARS
+
+
+# Define a set of characters we would expect to be replaced
+INVALID_URL_CHARS = [
+    chr(x) for x in range(127) if chr(x) not in LDH_ASCII_CHARS
+]
+for separator in [":", ".", "/", "#", "?", "@", "[", "]"]:
+    # Remove from the set characters that either separate hostname parts (":",
+    # "."), terminate hostnames ("/", "#", "?", "@"), or cause Python to be
+    # unable to parse URLs ("[", "]").
+    INVALID_URL_CHARS.remove(separator)
 
 
 class TestGetPackageMirrorInfo:
@@ -25,14 +36,16 @@ class TestGetPackageMirrorInfo:
         # Empty info gives empty return
         ({}, {}),
         # failsafe values used if present
-        ({'failsafe': {'primary': 'value', 'security': 'other'}},
-         {'primary': 'value', 'security': 'other'}),
+        ({'failsafe': {'primary': 'http://value', 'security': 'http://other'}},
+         {'primary': 'http://value', 'security': 'http://other'}),
         # search values used if present
-        ({'search': {'primary': ['value'], 'security': ['other']}},
-         {'primary': ['value'], 'security': ['other']}),
+        ({'search': {'primary': ['http://value'],
+                     'security': ['http://other']}},
+         {'primary': ['http://value'], 'security': ['http://other']}),
         # failsafe values used if search value not present
-        ({'search': {'primary': ['value']}, 'failsafe': {'security': 'other'}},
-         {'primary': ['value'], 'security': 'other'})
+        ({'search': {'primary': ['http://value']},
+          'failsafe': {'security': 'http://other'}},
+         {'primary': ['http://value'], 'security': 'http://other'})
     ])
     def test_get_package_mirror_info_failsafe(self, mirror_info, expected):
         """
@@ -48,26 +61,63 @@ class TestGetPackageMirrorInfo:
     def test_failsafe_used_if_all_search_results_filtered_out(self):
         """Test the failsafe option used if all search options eliminated."""
         mirror_info = {
-            'search': {'primary': ['value']}, 'failsafe': {'primary': 'other'}
+            'search': {'primary': ['http://value']},
+            'failsafe': {'primary': 'http://other'}
         }
-        assert {'primary': 'other'} == _get_package_mirror_info(
+        assert {'primary': 'http://other'} == _get_package_mirror_info(
             mirror_info, mirror_filter=lambda x: False)
 
     @pytest.mark.parametrize('availability_zone,region,patterns,expected', (
         # Test ec2_region alone
-        ('fk-fake-1f', None, ['EC2-%(ec2_region)s'], ['EC2-fk-fake-1']),
+        ('fk-fake-1f', None, ['http://EC2-%(ec2_region)s/ubuntu'],
+         ['http://ec2-fk-fake-1/ubuntu']),
         # Test availability_zone alone
-        ('fk-fake-1f', None, ['AZ-%(availability_zone)s'], ['AZ-fk-fake-1f']),
+        ('fk-fake-1f', None, ['http://AZ-%(availability_zone)s/ubuntu'],
+         ['http://az-fk-fake-1f/ubuntu']),
         # Test region alone
-        (None, 'fk-fake-1', ['RG-%(region)s'], ['RG-fk-fake-1']),
+        (None, 'fk-fake-1', ['http://RG-%(region)s/ubuntu'],
+         ['http://rg-fk-fake-1/ubuntu']),
         # Test that ec2_region is not available for non-matching AZs
         ('fake-fake-1f', None,
-         ['EC2-%(ec2_region)s', 'AZ-%(availability_zone)s'],
-         ['AZ-fake-fake-1f']),
+         ['http://EC2-%(ec2_region)s/ubuntu',
+          'http://AZ-%(availability_zone)s/ubuntu'],
+         ['http://az-fake-fake-1f/ubuntu']),
         # Test that template order maintained
-        (None, 'fake-region', ['RG-%(region)s-2', 'RG-%(region)s-1'],
-         ['RG-fake-region-2', 'RG-fake-region-1']),
-    ))
+        (None, 'fake-region',
+         ['http://RG-%(region)s-2/ubuntu', 'http://RG-%(region)s-1/ubuntu'],
+         ['http://rg-fake-region-2/ubuntu', 'http://rg-fake-region-1/ubuntu']),
+        # Test that non-ASCII hostnames are IDNA encoded;
+        # "IDNA-ТεЅТ̣".encode('idna') == b"xn--idna--4kd53hh6aba3q"
+        (None, 'ТεЅТ̣', ['http://www.IDNA-%(region)s.com/ubuntu'],
+         ['http://www.xn--idna--4kd53hh6aba3q.com/ubuntu']),
+        # Test that non-ASCII hostnames with a port are IDNA encoded;
+        # "IDNA-ТεЅТ̣".encode('idna') == b"xn--idna--4kd53hh6aba3q"
+        (None, 'ТεЅТ̣', ['http://www.IDNA-%(region)s.com:8080/ubuntu'],
+         ['http://www.xn--idna--4kd53hh6aba3q.com:8080/ubuntu']),
+        # Test that non-ASCII non-hostname parts of URLs are unchanged
+        (None, 'ТεЅТ̣', ['http://www.example.com/%(region)s/ubuntu'],
+         ['http://www.example.com/ТεЅТ̣/ubuntu']),
+        # Test that IPv4 addresses are unchanged
+        (None, 'fk-fake-1', ['http://192.168.1.1:8080/%(region)s/ubuntu'],
+         ['http://192.168.1.1:8080/fk-fake-1/ubuntu']),
+        # Test that IPv6 addresses are unchanged
+        (None, 'fk-fake-1',
+         ['http://[2001:67c:1360:8001::23]/%(region)s/ubuntu'],
+         ['http://[2001:67c:1360:8001::23]/fk-fake-1/ubuntu']),
+        # Test that unparseable URLs are filtered out of the mirror list
+        (None, 'inv[lid',
+         ['http://%(region)s.in.hostname/should/be/filtered',
+          'http://but.not.in.the.path/%(region)s'],
+         ['http://but.not.in.the.path/inv[lid']),
+    ) + (
+        # Dynamically generate a test case for each non-LDH
+        # (Letters/Digits/Hyphen) ASCII character, testing that it is
+        # substituted with a hyphen
+        tuple(
+            (None, 'fk{0}fake{0}1'.format(invalid_char),
+             ['http://%(region)s/ubuntu'], ['http://fk-fake-1/ubuntu'])
+            for invalid_char in INVALID_URL_CHARS))
+    )
     def test_substitution(self, availability_zone, region, patterns, expected):
         """Test substitution works as expected."""
         m_data_source = mock.Mock(
-- 
cgit v1.2.3


From 1bbc4908ff7a2be19483811b3b6fee6ebc916235 Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Tue, 31 Mar 2020 17:04:17 -0400
Subject: distros: drop leading/trailing hyphens from mirror URL labels (#296)

* distros/tests/test_init: drop needless brackets/indentation

* distros: drop leading/trailing hyphens from mirror URL labels
---
 cloudinit/distros/__init__.py        |  5 +++++
 cloudinit/distros/tests/test_init.py | 14 ++++++++------
 2 files changed, 13 insertions(+), 6 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 86a22c3e..e63b8c7a 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -816,6 +816,11 @@ def _sanitize_mirror_url(url: str):
         lambda hostname: ''.join(
             c if c in acceptable_chars else "-" for c in hostname
         ),
+
+        # Drop leading/trailing hyphens from each part of the hostname
+        lambda hostname: '.'.join(
+            part.strip('-') for part in hostname.split('.')
+        ),
     ]
 
     return _apply_hostname_transformations_to_url(url, transformations)
diff --git a/cloudinit/distros/tests/test_init.py b/cloudinit/distros/tests/test_init.py
index daa81ab8..40939133 100644
--- a/cloudinit/distros/tests/test_init.py
+++ b/cloudinit/distros/tests/test_init.py
@@ -109,15 +109,17 @@ class TestGetPackageMirrorInfo:
          ['http://%(region)s.in.hostname/should/be/filtered',
           'http://but.not.in.the.path/%(region)s'],
          ['http://but.not.in.the.path/inv[lid']),
-    ) + (
+        (None, '-some-region-',
+         ['http://-lead-ing.%(region)s.trail-ing-.example.com/ubuntu'],
+         ['http://lead-ing.some-region.trail-ing.example.com/ubuntu']),
+    ) + tuple(
         # Dynamically generate a test case for each non-LDH
         # (Letters/Digits/Hyphen) ASCII character, testing that it is
         # substituted with a hyphen
-        tuple(
-            (None, 'fk{0}fake{0}1'.format(invalid_char),
-             ['http://%(region)s/ubuntu'], ['http://fk-fake-1/ubuntu'])
-            for invalid_char in INVALID_URL_CHARS))
-    )
+        (None, 'fk{0}fake{0}1'.format(invalid_char),
+         ['http://%(region)s/ubuntu'], ['http://fk-fake-1/ubuntu'])
+        for invalid_char in INVALID_URL_CHARS
+    ))
     def test_substitution(self, availability_zone, region, patterns, expected):
         """Test substitution works as expected."""
         m_data_source = mock.Mock(
-- 
cgit v1.2.3


From bec8f387252ac32e1fd7963cf871ceed8d313edd Mon Sep 17 00:00:00 2001
From: Gonéri Le Bouder <goneri@lebouder.net>
Date: Thu, 2 Apr 2020 15:33:37 -0400
Subject: openbsd: set_passwd should not unlock user (#289)

Decouple unlocking passwords when also setting passwords. On OpenBSD skip unlocking
password as `usermode -C no foo` does not  work.
---
 cloudinit/distros/netbsd.py  | 10 +++++++++-
 cloudinit/distros/openbsd.py |  3 +++
 2 files changed, 12 insertions(+), 1 deletion(-)

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py
index 69d07846..6ae60943 100644
--- a/cloudinit/distros/netbsd.py
+++ b/cloudinit/distros/netbsd.py
@@ -95,10 +95,11 @@ class NetBSD(cloudinit.distros.bsd.BSD):
                     crypt.mksalt(method))
 
         try:
-            util.subp(['usermod', '-C', 'no', '-p', hashed_pw, user])
+            util.subp(['usermod', '-p', hashed_pw, user])
         except Exception:
             util.logexc(LOG, "Failed to set password for %s", user)
             raise
+        self.unlock_passwd(user)
 
     def force_passwd_change(self, user):
         try:
@@ -114,6 +115,13 @@ class NetBSD(cloudinit.distros.bsd.BSD):
             util.logexc(LOG, "Failed to lock user %s", name)
             raise
 
+    def unlock_passwd(self, name):
+        try:
+            util.subp(['usermod', '-C', 'no', name])
+        except Exception:
+            util.logexc(LOG, "Failed to unlock user %s", name)
+            raise
+
     def apply_locale(self, locale, out_fn=None):
         LOG.debug('Cannot set the locale.')
 
diff --git a/cloudinit/distros/openbsd.py b/cloudinit/distros/openbsd.py
index dbe1f069..ca094156 100644
--- a/cloudinit/distros/openbsd.py
+++ b/cloudinit/distros/openbsd.py
@@ -32,6 +32,9 @@ class Distro(cloudinit.distros.netbsd.NetBSD):
             util.logexc(LOG, "Failed to lock user %s", name)
             raise
 
+    def unlock_passwd(self, name):
+        pass
+
     def _get_pkg_cmd_environ(self):
         """Return env vars used in OpenBSD package_command operations"""
         os_release = platform.release()
-- 
cgit v1.2.3


From 7276aa5240b8cb84671a56d795d811f15dfba8e2 Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Thu, 23 Apr 2020 16:50:20 -0400
Subject: distros: handle a potential mirror filtering error case (#328)

As written, it's possible that the first transformation for a mirror
hostname could be passed None if the parsed mirror URL didn't have a
hostname component, when the defined interface is that the
transformations will be passed strings.

This isn't an error currently, because the first transformation happens
to gracefully handle being passed None.  It returns None, so the
pipeline processing ends there.

This was caught when testing out mypy on the cloud-init codebase.
---
 cloudinit/distros/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index e63b8c7a..e99529df 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -750,6 +750,10 @@ def _apply_hostname_transformations_to_url(url: str, transformations: list):
         # If we can't even parse the URL, we shouldn't use it for anything
         return None
     new_hostname = parts.hostname
+    if new_hostname is None:
+        # The URL given doesn't have a hostname component, so (a) we can't
+        # transform it, and (b) it won't work as a mirror; return None.
+        return None
 
     for transformation in transformations:
         new_hostname = transformation(new_hostname)
-- 
cgit v1.2.3


From f9b393bb4f15258de230884949e543e0f62f9abb Mon Sep 17 00:00:00 2001
From: Gonéri Le Bouder <goneri@lebouder.net>
Date: Mon, 4 May 2020 16:58:35 -0400
Subject: bsd: upgrade support (#305)

Implement the upgrade support:

- FreeBSD: using `pkg upgrade`
- NetBSD: with `pkgin`
---
 cloudinit/distros/bsd.py                    |  8 ++++++--
 cloudinit/distros/freebsd.py                |  1 +
 cloudinit/distros/netbsd.py                 | 14 +++++++++++---
 tests/unittests/test_distros/test_netbsd.py | 17 +++++++++++++++++
 4 files changed, 35 insertions(+), 5 deletions(-)
 create mode 100644 tests/unittests/test_distros/test_netbsd.py

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py
index efc73d5d..37cf93bf 100644
--- a/cloudinit/distros/bsd.py
+++ b/cloudinit/distros/bsd.py
@@ -18,9 +18,9 @@ class BSD(distros.Distro):
     group_add_cmd_prefix = []
     pkg_cmd_install_prefix = []
     pkg_cmd_remove_prefix = []
-    # There is no need to update the package cache on NetBSD and OpenBSD
-    # TODO neither freebsd nor netbsd handles a command 'upgrade'
+    # There is no update/upgrade on OpenBSD
     pkg_cmd_update_prefix = None
+    pkg_cmd_upgrade_prefix = None
 
     def __init__(self, name, cfg, paths):
         super().__init__(name, cfg, paths)
@@ -97,6 +97,10 @@ class BSD(distros.Distro):
             if not self.pkg_cmd_update_prefix:
                 return
             cmd = self.pkg_cmd_update_prefix
+        elif command == 'upgrade':
+            if not self.pkg_cmd_upgrade_prefix:
+                return
+            cmd = self.pkg_cmd_upgrade_prefix
 
         if args and isinstance(args, str):
             cmd.append(args)
diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py
index 71a6195b..b3a4ad67 100644
--- a/cloudinit/distros/freebsd.py
+++ b/cloudinit/distros/freebsd.py
@@ -25,6 +25,7 @@ class Distro(cloudinit.distros.bsd.BSD):
     pkg_cmd_install_prefix = ["pkg", "install"]
     pkg_cmd_remove_prefix = ["pkg", "remove"]
     pkg_cmd_update_prefix = ["pkg", "update"]
+    pkg_cmd_upgrade_prefix = ["pkg", "upgrade"]
 
     def _select_hostname(self, hostname, fqdn):
         # Should be FQDN if available. See rc.conf(5) in FreeBSD
diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py
index 6ae60943..ecc8239a 100644
--- a/cloudinit/distros/netbsd.py
+++ b/cloudinit/distros/netbsd.py
@@ -21,10 +21,18 @@ class NetBSD(cloudinit.distros.bsd.BSD):
     """
 
     ci_sudoers_fn = '/usr/pkg/etc/sudoers.d/90-cloud-init-users'
-
     group_add_cmd_prefix = ["groupadd"]
-    pkg_cmd_install_prefix = ["pkg_add", "-U"]
-    pkg_cmd_remove_prefix = ['pkg_delete']
+
+    def __init__(self, name, cfg, paths):
+        super().__init__(name, cfg, paths)
+        if os.path.exists("/usr/pkg/bin/pkgin"):
+            self.pkg_cmd_install_prefix = ['pkgin', '-y', 'install']
+            self.pkg_cmd_remove_prefix = ['pkgin', '-y', 'remove']
+            self.pkg_cmd_update_prefix = ['pkgin', '-y', 'update']
+            self.pkg_cmd_upgrade_prefix = ['pkgin', '-y', 'full-upgrade']
+        else:
+            self.pkg_cmd_install_prefix = ['pkg_add', '-U']
+            self.pkg_cmd_remove_prefix = ['pkg_delete']
 
     def _get_add_member_to_group_cmd(self, member_name, group_name):
         return ['usermod', '-G', group_name, member_name]
diff --git a/tests/unittests/test_distros/test_netbsd.py b/tests/unittests/test_distros/test_netbsd.py
new file mode 100644
index 00000000..11a68d2a
--- /dev/null
+++ b/tests/unittests/test_distros/test_netbsd.py
@@ -0,0 +1,17 @@
+import cloudinit.distros.netbsd
+
+import pytest
+import unittest.mock as mock
+
+
+@pytest.mark.parametrize('with_pkgin', (True, False))
+@mock.patch("cloudinit.distros.netbsd.os")
+def test_init(m_os, with_pkgin):
+    print(with_pkgin)
+    m_os.path.exists.return_value = with_pkgin
+    cfg = {}
+
+    distro = cloudinit.distros.netbsd.NetBSD("netbsd", cfg, None)
+    expectation = ['pkgin', '-y', 'full-upgrade'] if with_pkgin else None
+    assert distro.pkg_cmd_upgrade_prefix == expectation
+    assert [mock.call('/usr/pkg/bin/pkgin')] == m_os.path.exists.call_args_list
-- 
cgit v1.2.3


From b24b376fabd54e4dc96ffc1b90a984addf42fc4c Mon Sep 17 00:00:00 2001
From: chengcheng-chcheng
 <63850735+chengcheng-chcheng@users.noreply.github.com>
Date: Fri, 8 May 2020 01:01:22 +0800
Subject: make suse and sles support 127.0.1.1 (#336)

Move from 127.0.0.1 to 127.0.1.1 for localhost IP addr for opensuse and sles
---
 cloudinit/distros/opensuse.py                          | 3 +++
 templates/hosts.suse.tmpl                              | 2 +-
 tests/unittests/test_handler/test_handler_etc_hosts.py | 8 ++++----
 3 files changed, 8 insertions(+), 5 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py
index dd56a3f4..68028d20 100644
--- a/cloudinit/distros/opensuse.py
+++ b/cloudinit/distros/opensuse.py
@@ -144,6 +144,9 @@ class Distro(distros.Distro):
                 return default
             return hostname
 
+    def _get_localhost_ip(self):
+        return "127.0.1.1"
+
     def _read_hostname_conf(self, filename):
         conf = HostnameConf(util.load_file(filename))
         conf.parse()
diff --git a/templates/hosts.suse.tmpl b/templates/hosts.suse.tmpl
index 8e664dbf..5d7953f0 100644
--- a/templates/hosts.suse.tmpl
+++ b/templates/hosts.suse.tmpl
@@ -13,7 +13,7 @@ you need to add the following to config:
 #     /etc/cloud/cloud.cfg or cloud-config from user-data
 #
 # The following lines are desirable for IPv4 capable hosts
-127.0.0.1 {{fqdn}} {{hostname}}
+127.0.1.1 {{fqdn}} {{hostname}}
 127.0.0.1 localhost.localdomain localhost
 127.0.0.1 localhost4.localdomain4 localhost4
 
diff --git a/tests/unittests/test_handler/test_handler_etc_hosts.py b/tests/unittests/test_handler/test_handler_etc_hosts.py
index d854afcb..e3778b11 100644
--- a/tests/unittests/test_handler/test_handler_etc_hosts.py
+++ b/tests/unittests/test_handler/test_handler_etc_hosts.py
@@ -44,8 +44,8 @@ class TestHostsFile(t_help.FilesystemMockingTestCase):
         self.patchUtils(self.tmp)
         cc_update_etc_hosts.handle('test', cfg, cc, LOG, [])
         contents = util.load_file('%s/etc/hosts' % self.tmp)
-        if '127.0.0.1\tcloud-init.test.us\tcloud-init' not in contents:
-            self.assertIsNone('No entry for 127.0.0.1 in etc/hosts')
+        if '127.0.1.1\tcloud-init.test.us\tcloud-init' not in contents:
+            self.assertIsNone('No entry for 127.0.1.1 in etc/hosts')
         if '192.168.1.1\tblah.blah.us\tblah' not in contents:
             self.assertIsNone('Default etc/hosts content modified')
 
@@ -64,7 +64,7 @@ class TestHostsFile(t_help.FilesystemMockingTestCase):
         self.patchUtils(self.tmp)
         cc_update_etc_hosts.handle('test', cfg, cc, LOG, [])
         contents = util.load_file('%s/etc/hosts' % self.tmp)
-        if '127.0.0.1 cloud-init.test.us cloud-init' not in contents:
-            self.assertIsNone('No entry for 127.0.0.1 in etc/hosts')
+        if '127.0.1.1 cloud-init.test.us cloud-init' not in contents:
+            self.assertIsNone('No entry for 127.0.1.1 in etc/hosts')
         if '::1 cloud-init.test.us cloud-init' not in contents:
             self.assertIsNone('No entry for 127.0.0.1 in etc/hosts')
-- 
cgit v1.2.3


From 4ab3303ec4bb49f029b7821d6dba53a6b02b6dc1 Mon Sep 17 00:00:00 2001
From: Joshua Powers <josh.powers@canonical.com>
Date: Mon, 1 Jun 2020 14:20:39 -0700
Subject: test: fix all flake8 E741 errors (#401)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This removes the use of variables named ‘l’, ‘O’, or ‘I’. Generally
these are used in list comprehension to read the line of lines.
---
 cloudinit/distros/__init__.py                           |  2 +-
 cloudinit/net/__init__.py                               |  8 ++++----
 cloudinit/sources/DataSourceOpenNebula.py               |  6 +++---
 cloudinit/sources/helpers/openstack.py                  | 17 +++++++++++------
 cloudinit/ssh_util.py                                   |  4 +++-
 tests/cloud_tests/platforms/instances.py                |  4 ++--
 tests/cloud_tests/testcases/base.py                     |  4 ++--
 tests/unittests/test_datasource/test_ec2.py             |  4 +++-
 tests/unittests/test_datasource/test_ovf.py             |  2 +-
 tests/unittests/test_ds_identify.py                     |  6 ++++--
 .../test_handler/test_handler_apt_source_v3.py          |  4 +++-
 tests/unittests/test_sshutil.py                         |  2 +-
 tox.ini                                                 |  3 +--
 13 files changed, 39 insertions(+), 27 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index e99529df..35a10590 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -582,7 +582,7 @@ class Distro(metaclass=abc.ABCMeta):
         # passwd must use short '-l' due to SLES11 lacking long form '--lock'
         lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name])
         try:
-            cmd = next(l for l in lock_tools if util.which(l[0]))
+            cmd = next(tool for tool in lock_tools if util.which(tool[0]))
         except StopIteration:
             raise RuntimeError((
                 "Unable to lock user account '%s'. No tools available. "
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index cb8c1601..8af24fa9 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -824,13 +824,13 @@ def get_interfaces_by_mac_on_freebsd():
     # flatten each interface block in a single line
     def flatten(out):
         curr_block = ''
-        for l in out.split('\n'):
-            if l.startswith('\t'):
-                curr_block += l
+        for line in out.split('\n'):
+            if line.startswith('\t'):
+                curr_block += line
             else:
                 if curr_block:
                     yield curr_block
-                curr_block = l
+                curr_block = line
         yield curr_block
 
     # looks for interface and mac in a list of flatten block
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 02c9a7b8..a08ab404 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -417,9 +417,9 @@ def read_context_disk_dir(source_dir, asuser=None):
 
     if ssh_key_var:
         lines = context.get(ssh_key_var).splitlines()
-        results['metadata']['public-keys'] = [l for l in lines
-                                              if len(l) and not
-                                              l.startswith("#")]
+        results['metadata']['public-keys'] = [
+            line for line in lines if len(line) and not line.startswith("#")
+        ]
 
     # custom hostname -- try hostname or leave cloud-init
     # itself create hostname from IP address later
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index e91398ea..a4373f24 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -411,8 +411,11 @@ class ConfigDriveReader(BaseReader):
         keydata = meta_js.get('public-keys', keydata)
         if keydata:
             lines = keydata.splitlines()
-            md['public-keys'] = [l for l in lines
-                                 if len(l) and not l.startswith("#")]
+            md['public-keys'] = [
+                line
+                for line in lines
+                if len(line) and not line.startswith("#")
+            ]
 
         # config-drive-v1 has no way for openstack to provide the instance-id
         # so we copy that into metadata from the user input
@@ -674,11 +677,13 @@ def convert_net_json(network_json=None, known_macs=None):
                 raise ValueError("Unable to find a system nic for %s" % d)
             d['name'] = known_macs[mac]
 
-        for cfg, key, fmt, target in link_updates:
-            if isinstance(target, (list, tuple)):
-                cfg[key] = [fmt % link_id_info[l]['name'] for l in target]
+        for cfg, key, fmt, targets in link_updates:
+            if isinstance(targets, (list, tuple)):
+                cfg[key] = [
+                    fmt % link_id_info[target]['name'] for target in targets
+                ]
             else:
-                cfg[key] = fmt % link_id_info[target]['name']
+                cfg[key] = fmt % link_id_info[targets]['name']
 
     # Infiniband interfaces may be referenced in network_data.json by a 6 byte
     # Ethernet MAC-style address, and we use that address to look up the
diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py
index c3a9b5b7..918c4aec 100644
--- a/cloudinit/ssh_util.py
+++ b/cloudinit/ssh_util.py
@@ -344,7 +344,9 @@ def update_ssh_config(updates, fname=DEF_SSHD_CFG):
     changed = update_ssh_config_lines(lines=lines, updates=updates)
     if changed:
         util.write_file(
-            fname, "\n".join([str(l) for l in lines]) + "\n", copy_mode=True)
+            fname, "\n".join(
+                [str(line) for line in lines]
+            ) + "\n", copy_mode=True)
     return len(changed) != 0
 
 
diff --git a/tests/cloud_tests/platforms/instances.py b/tests/cloud_tests/platforms/instances.py
index 529e79cd..efc35c7f 100644
--- a/tests/cloud_tests/platforms/instances.py
+++ b/tests/cloud_tests/platforms/instances.py
@@ -132,8 +132,8 @@ class Instance(TargetBase):
         """
         def clean_test(test):
             """Clean formatting for system ready test testcase."""
-            return ' '.join(l for l in test.strip().splitlines()
-                            if not l.lstrip().startswith('#'))
+            return ' '.join(line for line in test.strip().splitlines()
+                            if not line.lstrip().startswith('#'))
 
         boot_timeout = self.config['boot_timeout']
         tests = [self.config['system_ready_script']]
diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index 7b67f54e..68d59111 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -141,8 +141,8 @@ class CloudTestCase(unittest.TestCase):
     def test_no_warnings_in_log(self):
         """Unexpected warnings should not be found in the log."""
         warnings = [
-            l for l in self.get_data_file('cloud-init.log').splitlines()
-            if 'WARN' in l]
+            line for line in self.get_data_file('cloud-init.log').splitlines()
+            if 'WARN' in line]
         joined_warnings = '\n'.join(warnings)
         for expected_warning in self.expected_warnings:
             self.assertIn(
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index d413d1cd..ad1ea595 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -612,7 +612,9 @@ class TestEc2(test_helpers.HttprettyTestCase):
         for log in expected_logs:
             self.assertIn(log, logs)
         self.assertEqual(
-            1, len([l for l in logs.splitlines() if failed_put_log in l]))
+            1,
+            len([line for line in logs.splitlines() if failed_put_log in line])
+        )
 
     def test_aws_token_redacted(self):
         """Verify that aws tokens are redacted when logged."""
diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py
index a19c35c8..486a2345 100644
--- a/tests/unittests/test_datasource/test_ovf.py
+++ b/tests/unittests/test_datasource/test_ovf.py
@@ -48,7 +48,7 @@ def fill_properties(props, template=OVF_ENV_CONTENT):
     for key, val in props.items():
         lines.append(prop_tmpl.format(key=key, val=val))
     indent = "        "
-    properties = ''.join([indent + l + "\n" for l in lines])
+    properties = ''.join([indent + line + "\n" for line in lines])
     return template.format(properties=properties)
 
 
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index f0e96b44..c2318570 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -611,8 +611,10 @@ class TestDsIdentify(DsIdentifyBase):
         ret = self._check_via_dict(
             cust, RC_FOUND,
             func=".", args=[os.path.join(rootd, mpp)], rootd=rootd)
-        line = [l for l in ret.stdout.splitlines() if l.startswith(pre)][0]
-        toks = line.replace(pre, "").split(":")
+        match = [
+            line for line in ret.stdout.splitlines() if line.startswith(pre)
+        ][0]
+        toks = match.replace(pre, "").split(":")
         expected = ["/sbin", "/bin", "/usr/sbin", "/usr/bin", "/mycust/path"]
         self.assertEqual(expected, [p for p in expected if p in toks],
                          "path did not have expected tokens")
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py
index 4762dbef..aefe26c4 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py
@@ -1033,7 +1033,9 @@ class TestDebconfSelections(TestCase):
         # assumes called with *args value.
         selections = m_set_sel.call_args_list[0][0][0].decode()
 
-        missing = [l for l in lines if l not in selections.splitlines()]
+        missing = [
+            line for line in lines if line not in selections.splitlines()
+        ]
         self.assertEqual([], missing)
 
     @mock.patch("cloudinit.config.cc_apt_configure.dpkg_reconfigure")
diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py
index 0be41924..b4767f0c 100644
--- a/tests/unittests/test_sshutil.py
+++ b/tests/unittests/test_sshutil.py
@@ -299,7 +299,7 @@ class TestUpdateSshConfigLines(test_helpers.CiTestCase):
         lines = ssh_util.parse_ssh_config_lines(list(self.exlines))
         result = ssh_util.update_ssh_config_lines(lines, updates)
         self.assertEqual([], result)
-        self.assertEqual(self.exlines, [str(l) for l in lines])
+        self.assertEqual(self.exlines, [str(line) for line in lines])
 
     def test_keycase_not_modified(self):
         """Original case of key should not be changed on update.
diff --git a/tox.ini b/tox.ini
index 611c3111..57b20568 100644
--- a/tox.ini
+++ b/tox.ini
@@ -49,10 +49,9 @@ deps = -r{toxinidir}/test-requirements.txt
 # E226: missing whitespace around arithmetic operator
 # E241: multiple spaces after ‘,’
 # E402: module level import not at top of file
-# E741: do not use variables named ‘l’, ‘O’, or ‘I’
 # W503: line break before binary operator
 # W504: line break after binary operator
-ignore=E121,E123,E126,E226,E241,E402,E741,W503,W504
+ignore=E121,E123,E126,E226,E241,E402,W503,W504
 exclude = .venv,.tox,dist,doc,*egg,.git,build,tools
 
 [testenv:doc]
-- 
cgit v1.2.3


From 3c551f6ebc12f7729a2755c89b19b9000e27cc88 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@brickies.net>
Date: Mon, 8 Jun 2020 12:49:12 -0400
Subject: Move subp into its own module. (#416)

This was painful, but it finishes a TODO from cloudinit/subp.py.

It moves the following from util to subp:
  ProcessExecutionError
  subp
  which
  target_path

I moved subp_blob_in_tempfile into cc_chef, which is its only caller.
That saved us from having to deal with it using write_file
and temp_utils from subp (which does not import any cloudinit things now).

It is arguable that 'target_path' could be moved to a 'path_utils' or
something, but in order to use it from subp and also from utils,
we had to get it out of utils.
---
 cloudinit/analyze/dump.py                          |   3 +-
 cloudinit/analyze/show.py                          |   5 +-
 cloudinit/analyze/tests/test_boot.py               |  16 +-
 cloudinit/analyze/tests/test_dump.py               |   3 +-
 cloudinit/cmd/clean.py                             |   5 +-
 cloudinit/cmd/devel/logs.py                        |   4 +-
 cloudinit/cmd/devel/tests/test_logs.py             |   3 +-
 cloudinit/config/cc_apt_configure.py               |  21 +-
 cloudinit/config/cc_bootcmd.py                     |   3 +-
 cloudinit/config/cc_byobu.py                       |   3 +-
 cloudinit/config/cc_ca_certs.py                    |   5 +-
 cloudinit/config/cc_chef.py                        |  35 +-
 cloudinit/config/cc_disable_ec2_metadata.py        |   7 +-
 cloudinit/config/cc_disk_setup.py                  |  45 +--
 cloudinit/config/cc_emit_upstart.py                |   8 +-
 cloudinit/config/cc_fan.py                         |   7 +-
 cloudinit/config/cc_growpart.py                    |  25 +-
 cloudinit/config/cc_grub_dpkg.py                   |   9 +-
 cloudinit/config/cc_keys_to_console.py             |   3 +-
 cloudinit/config/cc_landscape.py                   |   3 +-
 cloudinit/config/cc_lxd.py                         |  23 +-
 cloudinit/config/cc_mcollective.py                 |   3 +-
 cloudinit/config/cc_mounts.py                      |  15 +-
 cloudinit/config/cc_ntp.py                         |   9 +-
 .../config/cc_package_update_upgrade_install.py    |   3 +-
 cloudinit/config/cc_power_state_change.py          |   3 +-
 cloudinit/config/cc_puppet.py                      |   9 +-
 cloudinit/config/cc_resizefs.py                    |   9 +-
 cloudinit/config/cc_rh_subscription.py             |  19 +-
 cloudinit/config/cc_rsyslog.py                     |   5 +-
 cloudinit/config/cc_salt_minion.py                 |   4 +-
 cloudinit/config/cc_seed_random.py                 |   5 +-
 cloudinit/config/cc_set_passwords.py               |   5 +-
 cloudinit/config/cc_snap.py                        |   7 +-
 cloudinit/config/cc_spacewalk.py                   |   8 +-
 cloudinit/config/cc_ssh.py                         |   7 +-
 cloudinit/config/cc_ssh_import_id.py               |   5 +-
 cloudinit/config/cc_ubuntu_advantage.py            |  11 +-
 cloudinit/config/cc_ubuntu_drivers.py              |   9 +-
 .../config/tests/test_disable_ec2_metadata.py      |  12 +-
 cloudinit/config/tests/test_grub_dpkg.py           |   6 +-
 cloudinit/config/tests/test_mounts.py              |   4 +-
 cloudinit/config/tests/test_set_passwords.py       |  22 +-
 cloudinit/config/tests/test_snap.py                |  12 +-
 cloudinit/config/tests/test_ubuntu_advantage.py    |  28 +-
 cloudinit/config/tests/test_ubuntu_drivers.py      |  26 +-
 cloudinit/conftest.py                              |  28 +-
 cloudinit/distros/__init__.py                      |  25 +-
 cloudinit/distros/arch.py                          |  17 +-
 cloudinit/distros/bsd.py                           |   7 +-
 cloudinit/distros/debian.py                        |   9 +-
 cloudinit/distros/freebsd.py                       |  13 +-
 cloudinit/distros/gentoo.py                        |  17 +-
 cloudinit/distros/netbsd.py                        |  11 +-
 cloudinit/distros/openbsd.py                       |   3 +-
 cloudinit/distros/opensuse.py                      |   7 +-
 cloudinit/distros/rhel.py                          |   9 +-
 cloudinit/gpg.py                                   |  14 +-
 cloudinit/handlers/boot_hook.py                    |   5 +-
 cloudinit/handlers/upstart_job.py                  |  11 +-
 cloudinit/net/__init__.py                          |  37 +-
 cloudinit/net/bsd.py                               |  11 +-
 cloudinit/net/dhcp.py                              |   7 +-
 cloudinit/net/eni.py                               |   9 +-
 cloudinit/net/freebsd.py                           |   7 +-
 cloudinit/net/netbsd.py                            |   5 +-
 cloudinit/net/netplan.py                           |  17 +-
 cloudinit/net/openbsd.py                           |   7 +-
 cloudinit/net/sysconfig.py                         |  19 +-
 cloudinit/net/tests/test_dhcp.py                   |  12 +-
 cloudinit/net/tests/test_init.py                   |   5 +-
 cloudinit/netinfo.py                               |  27 +-
 cloudinit/sources/DataSourceAltCloud.py            |   8 +-
 cloudinit/sources/DataSourceAzure.py               |  21 +-
 cloudinit/sources/DataSourceCloudStack.py          |   3 +-
 cloudinit/sources/DataSourceConfigDrive.py         |   3 +-
 cloudinit/sources/DataSourceIBMCloud.py            |   3 +-
 cloudinit/sources/DataSourceOVF.py                 |   7 +-
 cloudinit/sources/DataSourceOpenNebula.py          |   5 +-
 cloudinit/sources/DataSourceRbxCloud.py            |   7 +-
 cloudinit/sources/DataSourceSmartOS.py             |   5 +-
 cloudinit/sources/helpers/azure.py                 |  17 +-
 cloudinit/sources/helpers/digitalocean.py          |   9 +-
 cloudinit/sources/helpers/openstack.py             |   3 +-
 .../helpers/vmware/imc/config_custom_script.py     |   3 +-
 cloudinit/sources/helpers/vmware/imc/config_nic.py |   7 +-
 .../sources/helpers/vmware/imc/config_passwd.py    |   7 +-
 .../sources/helpers/vmware/imc/guestcust_util.py   |  10 +-
 cloudinit/subp.py                                  | 301 +++++++++++++++-
 cloudinit/tests/helpers.py                         |  13 +-
 cloudinit/tests/test_conftest.py                   |  24 +-
 cloudinit/tests/test_gpg.py                        |  10 +-
 cloudinit/tests/test_netinfo.py                    |  40 +--
 cloudinit/tests/test_subp.py                       | 227 ++++++++++++-
 cloudinit/tests/test_util.py                       |  11 +-
 cloudinit/util.py                                  | 377 ++-------------------
 packages/bddeb                                     |   9 +-
 packages/brpm                                      |   5 +-
 tests/cloud_tests/bddeb.py                         |   6 +-
 tests/cloud_tests/platforms/lxd/image.py           |   9 +-
 tests/cloud_tests/platforms/lxd/instance.py        |   3 +-
 tests/cloud_tests/platforms/nocloudkvm/image.py    |  12 +-
 tests/cloud_tests/platforms/nocloudkvm/instance.py |  10 +-
 tests/cloud_tests/platforms/nocloudkvm/platform.py |   5 +-
 tests/cloud_tests/platforms/platforms.py           |   9 +-
 tests/cloud_tests/util.py                          |   7 +-
 tests/unittests/test_builtin_handlers.py           |   3 +-
 tests/unittests/test_datasource/test_altcloud.py   |   7 +-
 tests/unittests/test_datasource/test_azure.py      |  30 +-
 .../unittests/test_datasource/test_azure_helper.py |   2 +-
 tests/unittests/test_datasource/test_cloudstack.py |   2 +-
 tests/unittests/test_datasource/test_ovf.py        |   9 +-
 tests/unittests/test_datasource/test_rbx.py        |  10 +-
 tests/unittests/test_datasource/test_smartos.py    |   6 +-
 tests/unittests/test_distros/test_create_users.py  |   8 +-
 tests/unittests/test_distros/test_debian.py        |   2 +-
 tests/unittests/test_distros/test_freebsd.py       |   4 +-
 tests/unittests/test_distros/test_generic.py       |   4 +-
 tests/unittests/test_distros/test_netconfig.py     |   3 +-
 .../test_distros/test_user_data_normalize.py       |   6 +-
 tests/unittests/test_ds_identify.py                |   5 +-
 .../test_handler_apt_configure_sources_list_v1.py  |   5 +-
 .../test_handler_apt_configure_sources_list_v3.py  |   5 +-
 .../test_handler/test_handler_apt_source_v1.py     |  13 +-
 .../test_handler/test_handler_apt_source_v3.py     |  21 +-
 .../unittests/test_handler/test_handler_bootcmd.py |   6 +-
 .../test_handler/test_handler_ca_certs.py          |   5 +-
 tests/unittests/test_handler/test_handler_chef.py  |   6 +-
 .../test_handler/test_handler_disk_setup.py        |   8 +-
 .../test_handler/test_handler_growpart.py          |  10 +-
 .../test_handler/test_handler_landscape.py         |   6 +-
 .../unittests/test_handler/test_handler_locale.py  |   2 +-
 tests/unittests/test_handler/test_handler_lxd.py   |  28 +-
 .../test_handler/test_handler_mcollective.py       |   7 +-
 .../unittests/test_handler/test_handler_mounts.py  |   6 +-
 tests/unittests/test_handler/test_handler_ntp.py   |  51 +--
 .../unittests/test_handler/test_handler_puppet.py  |  26 +-
 .../unittests/test_handler/test_handler_runcmd.py  |   4 +-
 .../test_handler/test_handler_seed_random.py       |   5 +-
 .../test_handler/test_handler_spacewalk.py         |  20 +-
 tests/unittests/test_net.py                        |  33 +-
 tests/unittests/test_net_freebsd.py                |   2 +-
 tests/unittests/test_render_cloudcfg.py            |   7 +-
 tests/unittests/test_reporting_hyperv.py           |   2 +-
 tests/unittests/test_rh_subscription.py            |  18 +-
 tests/unittests/test_util.py                       | 258 ++------------
 tests/unittests/test_vmware/test_guestcust_util.py |  22 +-
 tools/.github-cla-signers                          |   1 +
 148 files changed, 1403 insertions(+), 1292 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/cloudinit/analyze/dump.py b/cloudinit/analyze/dump.py
index 939c3126..62ad51fe 100644
--- a/cloudinit/analyze/dump.py
+++ b/cloudinit/analyze/dump.py
@@ -4,6 +4,7 @@ import calendar
 from datetime import datetime
 import sys
 
+from cloudinit import subp
 from cloudinit import util
 
 stage_to_description = {
@@ -51,7 +52,7 @@ def parse_timestamp(timestampstr):
 
 
 def parse_timestamp_from_date(timestampstr):
-    out, _ = util.subp(['date', '+%s.%3N', '-d', timestampstr])
+    out, _ = subp.subp(['date', '+%s.%3N', '-d', timestampstr])
     timestamp = out.strip()
     return float(timestamp)
 
diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py
index fb152b1d..cca1fa7f 100644
--- a/cloudinit/analyze/show.py
+++ b/cloudinit/analyze/show.py
@@ -11,6 +11,7 @@ import os
 import time
 import sys
 
+from cloudinit import subp
 from cloudinit import util
 from cloudinit.distros import uses_systemd
 
@@ -155,7 +156,7 @@ class SystemctlReader(object):
         :return: whether the subp call failed or not
         '''
         try:
-            value, err = util.subp(self.args, capture=True)
+            value, err = subp.subp(self.args, capture=True)
             if err:
                 return err
             self.epoch = value
@@ -215,7 +216,7 @@ def gather_timestamps_using_dmesg():
     with gather_timestamps_using_systemd
     '''
     try:
-        data, _ = util.subp(['dmesg'], capture=True)
+        data, _ = subp.subp(['dmesg'], capture=True)
         split_entries = data[0].splitlines()
         for i in split_entries:
             if i.decode('UTF-8').find('user') != -1:
diff --git a/cloudinit/analyze/tests/test_boot.py b/cloudinit/analyze/tests/test_boot.py
index f4001c14..f69423c3 100644
--- a/cloudinit/analyze/tests/test_boot.py
+++ b/cloudinit/analyze/tests/test_boot.py
@@ -25,7 +25,7 @@ class TestDistroChecker(CiTestCase):
                                       m_get_linux_distro, m_is_FreeBSD):
         self.assertEqual(err_code, dist_check_timestamp())
 
-    @mock.patch('cloudinit.util.subp', return_value=(0, 1))
+    @mock.patch('cloudinit.subp.subp', return_value=(0, 1))
     def test_subp_fails(self, m_subp):
         self.assertEqual(err_code, dist_check_timestamp())
 
@@ -42,7 +42,7 @@ class TestSystemCtlReader(CiTestCase):
         with self.assertRaises(RuntimeError):
             reader.parse_epoch_as_float()
 
-    @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None))
     def test_systemctl_works_correctly_threshold(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         self.assertEqual(1.0, reader.parse_epoch_as_float())
@@ -50,12 +50,12 @@ class TestSystemCtlReader(CiTestCase):
         self.assertTrue(thresh < 1e-6)
         self.assertTrue(thresh > (-1 * 1e-6))
 
-    @mock.patch('cloudinit.util.subp', return_value=('U=0', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=0', None))
     def test_systemctl_succeed_zero(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         self.assertEqual(0.0, reader.parse_epoch_as_float())
 
-    @mock.patch('cloudinit.util.subp', return_value=('U=1', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=1', None))
     def test_systemctl_succeed_distinct(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         val1 = reader.parse_epoch_as_float()
@@ -64,13 +64,13 @@ class TestSystemCtlReader(CiTestCase):
         val2 = reader2.parse_epoch_as_float()
         self.assertNotEqual(val1, val2)
 
-    @mock.patch('cloudinit.util.subp', return_value=('100', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('100', None))
     def test_systemctl_epoch_not_splittable(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         with self.assertRaises(IndexError):
             reader.parse_epoch_as_float()
 
-    @mock.patch('cloudinit.util.subp', return_value=('U=foobar', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=foobar', None))
     def test_systemctl_cannot_convert_epoch_to_float(self, m_subp):
         reader = SystemctlReader('dummyProperty', 'dummyParameter')
         with self.assertRaises(ValueError):
@@ -130,7 +130,7 @@ class TestAnalyzeBoot(CiTestCase):
         self.assertEqual(err_string, data)
 
     @mock.patch("cloudinit.util.is_container", return_value=True)
-    @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None))
     def test_container_no_ci_log_line(self, m_is_container, m_subp):
         path = os.path.dirname(os.path.abspath(__file__))
         log_path = path + '/boot-test.log'
@@ -148,7 +148,7 @@ class TestAnalyzeBoot(CiTestCase):
         self.assertEqual(FAIL_CODE, finish_code)
 
     @mock.patch("cloudinit.util.is_container", return_value=True)
-    @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None))
+    @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None))
     @mock.patch('cloudinit.analyze.__main__._get_events', return_value=[{
         'name': 'init-local', 'description': 'starting search', 'timestamp':
         100000}])
diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py
index d6fbd381..dac1efb6 100644
--- a/cloudinit/analyze/tests/test_dump.py
+++ b/cloudinit/analyze/tests/test_dump.py
@@ -5,7 +5,8 @@ from textwrap import dedent
 
 from cloudinit.analyze.dump import (
     dump_events, parse_ci_logline, parse_timestamp)
-from cloudinit.util import which, write_file
+from cloudinit.util import write_file
+from cloudinit.subp import which
 from cloudinit.tests.helpers import CiTestCase, mock, skipIf
 
 
diff --git a/cloudinit/cmd/clean.py b/cloudinit/cmd/clean.py
index 30e49de0..928a8eea 100644
--- a/cloudinit/cmd/clean.py
+++ b/cloudinit/cmd/clean.py
@@ -10,9 +10,8 @@ import os
 import sys
 
 from cloudinit.stages import Init
-from cloudinit.util import (
-    ProcessExecutionError, del_dir, del_file, get_config_logfiles,
-    is_link, subp)
+from cloudinit.subp import (ProcessExecutionError, subp)
+from cloudinit.util import (del_dir, del_file, get_config_logfiles, is_link)
 
 
 def error(msg):
diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py
index 4c086b51..51c61cca 100644
--- a/cloudinit/cmd/devel/logs.py
+++ b/cloudinit/cmd/devel/logs.py
@@ -12,8 +12,8 @@ import sys
 
 from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE
 from cloudinit.temp_utils import tempdir
-from cloudinit.util import (
-    ProcessExecutionError, chdir, copy, ensure_dir, subp, write_file)
+from cloudinit.subp import (ProcessExecutionError, subp)
+from cloudinit.util import (chdir, copy, ensure_dir, write_file)
 
 
 CLOUDINIT_LOGS = ['/var/log/cloud-init.log', '/var/log/cloud-init-output.log']
diff --git a/cloudinit/cmd/devel/tests/test_logs.py b/cloudinit/cmd/devel/tests/test_logs.py
index d2dfa8de..ddfd58e1 100644
--- a/cloudinit/cmd/devel/tests/test_logs.py
+++ b/cloudinit/cmd/devel/tests/test_logs.py
@@ -8,7 +8,8 @@ from cloudinit.cmd.devel import logs
 from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE
 from cloudinit.tests.helpers import (
     FilesystemMockingTestCase, mock, wrap_and_call)
-from cloudinit.util import ensure_dir, load_file, subp, write_file
+from cloudinit.subp import subp
+from cloudinit.util import ensure_dir, load_file, write_file
 
 
 @mock.patch('cloudinit.cmd.devel.logs.os.getuid')
diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index b1c7b471..73d8719f 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -17,6 +17,7 @@ from cloudinit.config.schema import (
     get_schema_doc, validate_cloudconfig_schema)
 from cloudinit import gpg
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import templater
 from cloudinit import util
 from cloudinit.settings import PER_INSTANCE
@@ -431,7 +432,7 @@ def _should_configure_on_empty_apt():
     # if no config was provided, should apt configuration be done?
     if util.system_is_snappy():
         return False, "system is snappy."
-    if not (util.which('apt-get') or util.which('apt')):
+    if not (subp.which('apt-get') or subp.which('apt')):
         return False, "no apt commands."
     return True, "Apt is available."
 
@@ -478,7 +479,7 @@ def apply_apt(cfg, cloud, target):
 def debconf_set_selections(selections, target=None):
     if not selections.endswith(b'\n'):
         selections += b'\n'
-    util.subp(['debconf-set-selections'], data=selections, target=target,
+    subp.subp(['debconf-set-selections'], data=selections, target=target,
               capture=True)
 
 
@@ -503,7 +504,7 @@ def dpkg_reconfigure(packages, target=None):
                     "but cannot be unconfigured: %s", unhandled)
 
     if len(to_config):
-        util.subp(['dpkg-reconfigure', '--frontend=noninteractive'] +
+        subp.subp(['dpkg-reconfigure', '--frontend=noninteractive'] +
                   list(to_config), data=None, target=target, capture=True)
 
 
@@ -546,7 +547,7 @@ def apply_debconf_selections(cfg, target=None):
 def clean_cloud_init(target):
     """clean out any local cloud-init config"""
     flist = glob.glob(
-        util.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*"))
+        subp.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*"))
 
     LOG.debug("cleaning cloud-init config from: %s", flist)
     for dpkg_cfg in flist:
@@ -575,7 +576,7 @@ def rename_apt_lists(new_mirrors, target, arch):
     """rename_apt_lists - rename apt lists to preserve old cache data"""
     default_mirrors = get_default_mirrors(arch)
 
-    pre = util.target_path(target, APT_LISTS)
+    pre = subp.target_path(target, APT_LISTS)
     for (name, omirror) in default_mirrors.items():
         nmirror = new_mirrors.get(name)
         if not nmirror:
@@ -694,8 +695,8 @@ def add_apt_key_raw(key, target=None):
     """
     LOG.debug("Adding key:\n'%s'", key)
     try:
-        util.subp(['apt-key', 'add', '-'], data=key.encode(), target=target)
-    except util.ProcessExecutionError:
+        subp.subp(['apt-key', 'add', '-'], data=key.encode(), target=target)
+    except subp.ProcessExecutionError:
         LOG.exception("failed to add apt GPG Key to apt keyring")
         raise
 
@@ -758,13 +759,13 @@ def add_apt_sources(srcdict, cloud, target=None, template_params=None,
 
         if aa_repo_match(source):
             try:
-                util.subp(["add-apt-repository", source], target=target)
-            except util.ProcessExecutionError:
+                subp.subp(["add-apt-repository", source], target=target)
+            except subp.ProcessExecutionError:
                 LOG.exception("add-apt-repository failed.")
                 raise
             continue
 
-        sourcefn = util.target_path(target, ent['filename'])
+        sourcefn = subp.target_path(target, ent['filename'])
         try:
             contents = "%s\n" % (source)
             util.write_file(sourcefn, contents, omode="a")
diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py
index 6813f534..246e4497 100644
--- a/cloudinit/config/cc_bootcmd.py
+++ b/cloudinit/config/cc_bootcmd.py
@@ -16,6 +16,7 @@ from cloudinit.config.schema import (
     get_schema_doc, validate_cloudconfig_schema)
 from cloudinit.settings import PER_ALWAYS
 from cloudinit import temp_utils
+from cloudinit import subp
 from cloudinit import util
 
 frequency = PER_ALWAYS
@@ -99,7 +100,7 @@ def handle(name, cfg, cloud, log, _args):
             if iid:
                 env['INSTANCE_ID'] = str(iid)
             cmd = ['/bin/sh', tmpf.name]
-            util.subp(cmd, env=env, capture=False)
+            subp.subp(cmd, env=env, capture=False)
         except Exception:
             util.logexc(log, "Failed to run bootcmd module %s", name)
             raise
diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py
index 0b4352c8..9fdaeba1 100755
--- a/cloudinit/config/cc_byobu.py
+++ b/cloudinit/config/cc_byobu.py
@@ -39,6 +39,7 @@ Valid configuration options for this module are:
 """
 
 from cloudinit.distros import ug_util
+from cloudinit import subp
 from cloudinit import util
 
 distros = ['ubuntu', 'debian']
@@ -93,6 +94,6 @@ def handle(name, cfg, cloud, log, args):
     if len(shcmd):
         cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")]
         log.debug("Setting byobu to %s", value)
-        util.subp(cmd, capture=False)
+        subp.subp(cmd, capture=False)
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py
index 64bc900e..7617a8ea 100644
--- a/cloudinit/config/cc_ca_certs.py
+++ b/cloudinit/config/cc_ca_certs.py
@@ -36,6 +36,7 @@ can be removed from the system with the configuration option
 
 import os
 
+from cloudinit import subp
 from cloudinit import util
 
 CA_CERT_PATH = "/usr/share/ca-certificates/"
@@ -51,7 +52,7 @@ def update_ca_certs():
     """
     Updates the CA certificate cache on the current machine.
     """
-    util.subp(["update-ca-certificates"], capture=False)
+    subp.subp(["update-ca-certificates"], capture=False)
 
 
 def add_ca_certs(certs):
@@ -85,7 +86,7 @@ def remove_default_ca_certs():
     util.delete_dir_contents(CA_CERT_SYSTEM_PATH)
     util.write_file(CA_CERT_CONFIG, "", mode=0o644)
     debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no"
-    util.subp(('debconf-set-selections', '-'), debconf_sel)
+    subp.subp(('debconf-set-selections', '-'), debconf_sel)
 
 
 def handle(name, cfg, _cloud, log, _args):
diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py
index 03285ef0..e1f51fce 100644
--- a/cloudinit/config/cc_chef.py
+++ b/cloudinit/config/cc_chef.py
@@ -76,7 +76,9 @@ import itertools
 import json
 import os
 
+from cloudinit import subp
 from cloudinit import templater
+from cloudinit import temp_utils
 from cloudinit import url_helper
 from cloudinit import util
 
@@ -282,7 +284,32 @@ def run_chef(chef_cfg, log):
             cmd.extend(CHEF_EXEC_DEF_ARGS)
     else:
         cmd.extend(CHEF_EXEC_DEF_ARGS)
-    util.subp(cmd, capture=False)
+    subp.subp(cmd, capture=False)
+
+
+def subp_blob_in_tempfile(blob, *args, **kwargs):
+    """Write blob to a tempfile, and call subp with args, kwargs. Then cleanup.
+
+    'basename' as a kwarg allows providing the basename for the file.
+    The 'args' argument to subp will be updated with the full path to the
+    filename as the first argument.
+    """
+    basename = kwargs.pop('basename', "subp_blob")
+
+    if len(args) == 0 and 'args' not in kwargs:
+        args = [tuple()]
+
+    # Use tmpdir over tmpfile to avoid 'text file busy' on execute
+    with temp_utils.tempdir(needs_exe=True) as tmpd:
+        tmpf = os.path.join(tmpd, basename)
+        if 'args' in kwargs:
+            kwargs['args'] = [tmpf] + list(kwargs['args'])
+        else:
+            args = list(args)
+            args[0] = [tmpf] + args[0]
+
+        util.write_file(tmpf, blob, mode=0o700)
+        return subp.subp(*args, **kwargs)
 
 
 def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None):
@@ -305,7 +332,7 @@ def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None):
     else:
         args = ['-v', omnibus_version]
     content = url_helper.readurl(url=url, retries=retries).contents
-    return util.subp_blob_in_tempfile(
+    return subp_blob_in_tempfile(
         blob=content, args=args,
         basename='chef-omnibus-install', capture=False)
 
@@ -354,11 +381,11 @@ def install_chef_from_gems(ruby_version, chef_version, distro):
     if not os.path.exists('/usr/bin/ruby'):
         util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby')
     if chef_version:
-        util.subp(['/usr/bin/gem', 'install', 'chef',
+        subp.subp(['/usr/bin/gem', 'install', 'chef',
                    '-v %s' % chef_version, '--no-ri',
                    '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False)
     else:
-        util.subp(['/usr/bin/gem', 'install', 'chef',
+        subp.subp(['/usr/bin/gem', 'install', 'chef',
                    '--no-ri', '--no-rdoc', '--bindir',
                    '/usr/bin', '-q'], capture=False)
 
diff --git a/cloudinit/config/cc_disable_ec2_metadata.py b/cloudinit/config/cc_disable_ec2_metadata.py
index 885b3138..dff93245 100644
--- a/cloudinit/config/cc_disable_ec2_metadata.py
+++ b/cloudinit/config/cc_disable_ec2_metadata.py
@@ -26,6 +26,7 @@ by default.
     disable_ec2_metadata: <true/false>
 """
 
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.settings import PER_ALWAYS
@@ -40,15 +41,15 @@ def handle(name, cfg, _cloud, log, _args):
     disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False)
     if disabled:
         reject_cmd = None
-        if util.which('ip'):
+        if subp.which('ip'):
             reject_cmd = REJECT_CMD_IP
-        elif util.which('ifconfig'):
+        elif subp.which('ifconfig'):
             reject_cmd = REJECT_CMD_IF
         else:
             log.error(('Neither "route" nor "ip" command found, unable to '
                        'manipulate routing table'))
             return
-        util.subp(reject_cmd, capture=False)
+        subp.subp(reject_cmd, capture=False)
     else:
         log.debug(("Skipping module named %s,"
                    " disabling the ec2 route not enabled"), name)
diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py
index 45925755..d957cfe3 100644
--- a/cloudinit/config/cc_disk_setup.py
+++ b/cloudinit/config/cc_disk_setup.py
@@ -99,6 +99,7 @@ specified using ``filesystem``.
 
 from cloudinit.settings import PER_INSTANCE
 from cloudinit import util
+from cloudinit import subp
 import logging
 import os
 import shlex
@@ -106,13 +107,13 @@ import shlex
 frequency = PER_INSTANCE
 
 # Define the commands to use
-UDEVADM_CMD = util.which('udevadm')
-SFDISK_CMD = util.which("sfdisk")
-SGDISK_CMD = util.which("sgdisk")
-LSBLK_CMD = util.which("lsblk")
-BLKID_CMD = util.which("blkid")
-BLKDEV_CMD = util.which("blockdev")
-WIPEFS_CMD = util.which("wipefs")
+UDEVADM_CMD = subp.which('udevadm')
+SFDISK_CMD = subp.which("sfdisk")
+SGDISK_CMD = subp.which("sgdisk")
+LSBLK_CMD = subp.which("lsblk")
+BLKID_CMD = subp.which("blkid")
+BLKDEV_CMD = subp.which("blockdev")
+WIPEFS_CMD = subp.which("wipefs")
 
 LANG_C_ENV = {'LANG': 'C'}
 
@@ -248,7 +249,7 @@ def enumerate_disk(device, nodeps=False):
 
     info = None
     try:
-        info, _err = util.subp(lsblk_cmd)
+        info, _err = subp.subp(lsblk_cmd)
     except Exception as e:
         raise Exception("Failed during disk check for %s\n%s" % (device, e))
 
@@ -310,7 +311,7 @@ def check_fs(device):
 
     blkid_cmd = [BLKID_CMD, '-c', '/dev/null', device]
     try:
-        out, _err = util.subp(blkid_cmd, rcs=[0, 2])
+        out, _err = subp.subp(blkid_cmd, rcs=[0, 2])
     except Exception as e:
         raise Exception("Failed during disk check for %s\n%s" % (device, e))
 
@@ -433,8 +434,8 @@ def get_dyn_func(*args):
 
 def get_hdd_size(device):
     try:
-        size_in_bytes, _ = util.subp([BLKDEV_CMD, '--getsize64', device])
-        sector_size, _ = util.subp([BLKDEV_CMD, '--getss', device])
+        size_in_bytes, _ = subp.subp([BLKDEV_CMD, '--getsize64', device])
+        sector_size, _ = subp.subp([BLKDEV_CMD, '--getss', device])
     except Exception as e:
         raise Exception("Failed to get %s size\n%s" % (device, e))
 
@@ -452,7 +453,7 @@ def check_partition_mbr_layout(device, layout):
     read_parttbl(device)
     prt_cmd = [SFDISK_CMD, "-l", device]
     try:
-        out, _err = util.subp(prt_cmd, data="%s\n" % layout)
+        out, _err = subp.subp(prt_cmd, data="%s\n" % layout)
     except Exception as e:
         raise Exception("Error running partition command on %s\n%s" % (
                         device, e))
@@ -482,7 +483,7 @@ def check_partition_mbr_layout(device, layout):
 def check_partition_gpt_layout(device, layout):
     prt_cmd = [SGDISK_CMD, '-p', device]
     try:
-        out, _err = util.subp(prt_cmd, update_env=LANG_C_ENV)
+        out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV)
     except Exception as e:
         raise Exception("Error running partition command on %s\n%s" % (
                         device, e))
@@ -655,7 +656,7 @@ def purge_disk(device):
             wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']]
             try:
                 LOG.info("Purging filesystem on /dev/%s", d['name'])
-                util.subp(wipefs_cmd)
+                subp.subp(wipefs_cmd)
             except Exception:
                 raise Exception("Failed FS purge of /dev/%s" % d['name'])
 
@@ -682,7 +683,7 @@ def read_parttbl(device):
     blkdev_cmd = [BLKDEV_CMD, '--rereadpt', device]
     util.udevadm_settle()
     try:
-        util.subp(blkdev_cmd)
+        subp.subp(blkdev_cmd)
     except Exception as e:
         util.logexc(LOG, "Failed reading the partition table %s" % e)
 
@@ -697,7 +698,7 @@ def exec_mkpart_mbr(device, layout):
     # Create the partitions
     prt_cmd = [SFDISK_CMD, "--Linux", "--unit=S", "--force", device]
     try:
-        util.subp(prt_cmd, data="%s\n" % layout)
+        subp.subp(prt_cmd, data="%s\n" % layout)
     except Exception as e:
         raise Exception("Failed to partition device %s\n%s" % (device, e))
 
@@ -706,16 +707,16 @@ def exec_mkpart_mbr(device, layout):
 
 def exec_mkpart_gpt(device, layout):
     try:
-        util.subp([SGDISK_CMD, '-Z', device])
+        subp.subp([SGDISK_CMD, '-Z', device])
         for index, (partition_type, (start, end)) in enumerate(layout):
             index += 1
-            util.subp([SGDISK_CMD,
+            subp.subp([SGDISK_CMD,
                        '-n', '{}:{}:{}'.format(index, start, end), device])
             if partition_type is not None:
                 # convert to a 4 char (or more) string right padded with 0
                 # 82 -> 8200.  'Linux' -> 'Linux'
                 pinput = str(partition_type).ljust(4, "0")
-                util.subp(
+                subp.subp(
                     [SGDISK_CMD, '-t', '{}:{}'.format(index, pinput), device])
     except Exception:
         LOG.warning("Failed to partition device %s", device)
@@ -967,9 +968,9 @@ def mkfs(fs_cfg):
                 fs_cmd)
     else:
         # Find the mkfs command
-        mkfs_cmd = util.which("mkfs.%s" % fs_type)
+        mkfs_cmd = subp.which("mkfs.%s" % fs_type)
         if not mkfs_cmd:
-            mkfs_cmd = util.which("mk%s" % fs_type)
+            mkfs_cmd = subp.which("mk%s" % fs_type)
 
         if not mkfs_cmd:
             LOG.warning("Cannot create fstype '%s'.  No mkfs.%s command",
@@ -994,7 +995,7 @@ def mkfs(fs_cfg):
     LOG.debug("Creating file system %s on %s", label, device)
     LOG.debug("     Using cmd: %s", str(fs_cmd))
     try:
-        util.subp(fs_cmd, shell=shell)
+        subp.subp(fs_cmd, shell=shell)
     except Exception as e:
         raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e))
 
diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py
index b342e04d..b1d99f97 100644
--- a/cloudinit/config/cc_emit_upstart.py
+++ b/cloudinit/config/cc_emit_upstart.py
@@ -25,7 +25,7 @@ import os
 
 from cloudinit import log as logging
 from cloudinit.settings import PER_ALWAYS
-from cloudinit import util
+from cloudinit import subp
 
 frequency = PER_ALWAYS
 
@@ -43,9 +43,9 @@ def is_upstart_system():
         del myenv['UPSTART_SESSION']
     check_cmd = ['initctl', 'version']
     try:
-        (out, _err) = util.subp(check_cmd, env=myenv)
+        (out, _err) = subp.subp(check_cmd, env=myenv)
         return 'upstart' in out
-    except util.ProcessExecutionError as e:
+    except subp.ProcessExecutionError as e:
         LOG.debug("'%s' returned '%s', not using upstart",
                   ' '.join(check_cmd), e.exit_code)
     return False
@@ -66,7 +66,7 @@ def handle(name, _cfg, cloud, log, args):
     for n in event_names:
         cmd = ['initctl', 'emit', str(n), 'CLOUD_CFG=%s' % cfgpath]
         try:
-            util.subp(cmd)
+            subp.subp(cmd)
         except Exception as e:
             # TODO(harlowja), use log exception from utils??
             log.warning("Emission of upstart event %s failed due to: %s", n, e)
diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py
index 0a135bbe..77984bca 100644
--- a/cloudinit/config/cc_fan.py
+++ b/cloudinit/config/cc_fan.py
@@ -39,6 +39,7 @@ If cloud-init sees a ``fan`` entry in cloud-config it will:
 
 from cloudinit import log as logging
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import subp
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -62,8 +63,8 @@ def stop_update_start(service, config_file, content, systemd=False):
 
     def run(cmd, msg):
         try:
-            return util.subp(cmd, capture=True)
-        except util.ProcessExecutionError as e:
+            return subp.subp(cmd, capture=True)
+        except subp.ProcessExecutionError as e:
             LOG.warning("failed: %s (%s): %s", service, cmd, e)
             return False
 
@@ -94,7 +95,7 @@ def handle(name, cfg, cloud, log, args):
 
     util.write_file(mycfg.get('config_path'), mycfg.get('config'), omode="w")
     distro = cloud.distro
-    if not util.which('fanctl'):
+    if not subp.which('fanctl'):
         distro.install_packages(['ubuntu-fan'])
 
     stop_update_start(
diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py
index 1b512a06..c5d93f81 100644
--- a/cloudinit/config/cc_growpart.py
+++ b/cloudinit/config/cc_growpart.py
@@ -70,6 +70,7 @@ import stat
 
 from cloudinit import log as logging
 from cloudinit.settings import PER_ALWAYS
+from cloudinit import subp
 from cloudinit import util
 
 frequency = PER_ALWAYS
@@ -131,19 +132,19 @@ class ResizeGrowPart(object):
         myenv['LANG'] = 'C'
 
         try:
-            (out, _err) = util.subp(["growpart", "--help"], env=myenv)
+            (out, _err) = subp.subp(["growpart", "--help"], env=myenv)
             if re.search(r"--update\s+", out):
                 return True
 
-        except util.ProcessExecutionError:
+        except subp.ProcessExecutionError:
             pass
         return False
 
     def resize(self, diskdev, partnum, partdev):
         before = get_size(partdev)
         try:
-            util.subp(["growpart", '--dry-run', diskdev, partnum])
-        except util.ProcessExecutionError as e:
+            subp.subp(["growpart", '--dry-run', diskdev, partnum])
+        except subp.ProcessExecutionError as e:
             if e.exit_code != 1:
                 util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)",
                             diskdev, partnum)
@@ -151,8 +152,8 @@ class ResizeGrowPart(object):
             return (before, before)
 
         try:
-            util.subp(["growpart", diskdev, partnum])
-        except util.ProcessExecutionError as e:
+            subp.subp(["growpart", diskdev, partnum])
+        except subp.ProcessExecutionError as e:
             util.logexc(LOG, "Failed: growpart %s %s", diskdev, partnum)
             raise ResizeFailedException(e)
 
@@ -165,11 +166,11 @@ class ResizeGpart(object):
         myenv['LANG'] = 'C'
 
         try:
-            (_out, err) = util.subp(["gpart", "help"], env=myenv, rcs=[0, 1])
+            (_out, err) = subp.subp(["gpart", "help"], env=myenv, rcs=[0, 1])
             if re.search(r"gpart recover ", err):
                 return True
 
-        except util.ProcessExecutionError:
+        except subp.ProcessExecutionError:
             pass
         return False
 
@@ -182,16 +183,16 @@ class ResizeGpart(object):
         be recovered.
         """
         try:
-            util.subp(["gpart", "recover", diskdev])
-        except util.ProcessExecutionError as e:
+            subp.subp(["gpart", "recover", diskdev])
+        except subp.ProcessExecutionError as e:
             if e.exit_code != 0:
                 util.logexc(LOG, "Failed: gpart recover %s", diskdev)
                 raise ResizeFailedException(e)
 
         before = get_size(partdev)
         try:
-            util.subp(["gpart", "resize", "-i", partnum, diskdev])
-        except util.ProcessExecutionError as e:
+            subp.subp(["gpart", "resize", "-i", partnum, diskdev])
+        except subp.ProcessExecutionError as e:
             util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev)
             raise ResizeFailedException(e)
 
diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py
index 7888464e..eb03c664 100644
--- a/cloudinit/config/cc_grub_dpkg.py
+++ b/cloudinit/config/cc_grub_dpkg.py
@@ -43,8 +43,9 @@ seeded with empty values, and install_devices_empty is set to true.
 
 import os
 
+from cloudinit import subp
 from cloudinit import util
-from cloudinit.util import ProcessExecutionError
+from cloudinit.subp import ProcessExecutionError
 
 distros = ['ubuntu', 'debian']
 
@@ -59,7 +60,7 @@ def fetch_idevs(log):
 
     try:
         # get the root disk where the /boot directory resides.
-        disk = util.subp(['grub-probe', '-t', 'disk', '/boot'],
+        disk = subp.subp(['grub-probe', '-t', 'disk', '/boot'],
                          capture=True)[0].strip()
     except ProcessExecutionError as e:
         # grub-common may not be installed, especially on containers
@@ -84,7 +85,7 @@ def fetch_idevs(log):
 
     try:
         # check if disk exists and use udevadm to fetch symlinks
-        devices = util.subp(
+        devices = subp.subp(
             ['udevadm', 'info', '--root', '--query=symlink', disk],
             capture=True
         )[0].strip().split()
@@ -135,7 +136,7 @@ def handle(name, cfg, _cloud, log, _args):
               (idevs, idevs_empty))
 
     try:
-        util.subp(['debconf-set-selections'], dconf_sel)
+        subp.subp(['debconf-set-selections'], dconf_sel)
     except Exception:
         util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg")
 
diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py
index 3d2ded3d..0f2be52b 100644
--- a/cloudinit/config/cc_keys_to_console.py
+++ b/cloudinit/config/cc_keys_to_console.py
@@ -33,6 +33,7 @@ key can be used. By default ``ssh-dss`` keys are not written to console.
 import os
 
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import subp
 from cloudinit import util
 
 frequency = PER_INSTANCE
@@ -64,7 +65,7 @@ def handle(name, cfg, cloud, log, _args):
 
     try:
         cmd = [helper_path, ','.join(fp_blacklist), ','.join(key_blacklist)]
-        (stdout, _stderr) = util.subp(cmd)
+        (stdout, _stderr) = subp.subp(cmd)
         util.multi_log("%s\n" % (stdout.strip()),
                        stderr=False, console=True)
     except Exception:
diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py
index a9c04d86..299c4d01 100644
--- a/cloudinit/config/cc_landscape.py
+++ b/cloudinit/config/cc_landscape.py
@@ -61,6 +61,7 @@ from io import BytesIO
 from configobj import ConfigObj
 
 from cloudinit import type_utils
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.settings import PER_INSTANCE
@@ -116,7 +117,7 @@ def handle(_name, cfg, cloud, log, _args):
     log.debug("Wrote landscape config file to %s", LSC_CLIENT_CFG_FILE)
 
     util.write_file(LS_DEFAULT_FILE, "RUN=1\n")
-    util.subp(["service", "landscape-client", "restart"])
+    subp.subp(["service", "landscape-client", "restart"])
 
 
 def merge_together(objs):
diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py
index 151a9844..7129c9c6 100644
--- a/cloudinit/config/cc_lxd.py
+++ b/cloudinit/config/cc_lxd.py
@@ -48,6 +48,7 @@ lxd-bridge will be configured accordingly.
 """
 
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 import os
 
@@ -85,16 +86,16 @@ def handle(name, cfg, cloud, log, args):
 
     # Install the needed packages
     packages = []
-    if not util.which("lxd"):
+    if not subp.which("lxd"):
         packages.append('lxd')
 
-    if init_cfg.get("storage_backend") == "zfs" and not util.which('zfs'):
+    if init_cfg.get("storage_backend") == "zfs" and not subp.which('zfs'):
         packages.append('zfsutils-linux')
 
     if len(packages):
         try:
             cloud.distro.install_packages(packages)
-        except util.ProcessExecutionError as exc:
+        except subp.ProcessExecutionError as exc:
             log.warning("failed to install packages %s: %s", packages, exc)
             return
 
@@ -104,20 +105,20 @@ def handle(name, cfg, cloud, log, args):
             'network_address', 'network_port', 'storage_backend',
             'storage_create_device', 'storage_create_loop',
             'storage_pool', 'trust_password')
-        util.subp(['lxd', 'waitready', '--timeout=300'])
+        subp.subp(['lxd', 'waitready', '--timeout=300'])
         cmd = ['lxd', 'init', '--auto']
         for k in init_keys:
             if init_cfg.get(k):
                 cmd.extend(["--%s=%s" %
                             (k.replace('_', '-'), str(init_cfg[k]))])
-        util.subp(cmd)
+        subp.subp(cmd)
 
     # Set up lxd-bridge if bridge config is given
     dconf_comm = "debconf-communicate"
     if bridge_cfg:
         net_name = bridge_cfg.get("name", _DEFAULT_NETWORK_NAME)
         if os.path.exists("/etc/default/lxd-bridge") \
-                and util.which(dconf_comm):
+                and subp.which(dconf_comm):
             # Bridge configured through packaging
 
             debconf = bridge_to_debconf(bridge_cfg)
@@ -127,7 +128,7 @@ def handle(name, cfg, cloud, log, args):
                 log.debug("Setting lxd debconf via " + dconf_comm)
                 data = "\n".join(["set %s %s" % (k, v)
                                   for k, v in debconf.items()]) + "\n"
-                util.subp(['debconf-communicate'], data)
+                subp.subp(['debconf-communicate'], data)
             except Exception:
                 util.logexc(log, "Failed to run '%s' for lxd with" %
                             dconf_comm)
@@ -137,7 +138,7 @@ def handle(name, cfg, cloud, log, args):
 
             # Run reconfigure
             log.debug("Running dpkg-reconfigure for lxd")
-            util.subp(['dpkg-reconfigure', 'lxd',
+            subp.subp(['dpkg-reconfigure', 'lxd',
                        '--frontend=noninteractive'])
         else:
             # Built-in LXD bridge support
@@ -264,7 +265,7 @@ def _lxc(cmd):
     env = {'LC_ALL': 'C',
            'HOME': os.environ.get('HOME', '/root'),
            'USER': os.environ.get('USER', 'root')}
-    util.subp(['lxc'] + list(cmd) + ["--force-local"], update_env=env)
+    subp.subp(['lxc'] + list(cmd) + ["--force-local"], update_env=env)
 
 
 def maybe_cleanup_default(net_name, did_init, create, attach,
@@ -286,7 +287,7 @@ def maybe_cleanup_default(net_name, did_init, create, attach,
         try:
             _lxc(["network", "delete", net_name])
             LOG.debug(msg, net_name, succeeded)
-        except util.ProcessExecutionError as e:
+        except subp.ProcessExecutionError as e:
             if e.exit_code != 1:
                 raise e
             LOG.debug(msg, net_name, fail_assume_enoent)
@@ -296,7 +297,7 @@ def maybe_cleanup_default(net_name, did_init, create, attach,
         try:
             _lxc(["profile", "device", "remove", profile, nic_name])
             LOG.debug(msg, nic_name, profile, succeeded)
-        except util.ProcessExecutionError as e:
+        except subp.ProcessExecutionError as e:
             if e.exit_code != 1:
                 raise e
             LOG.debug(msg, nic_name, profile, fail_assume_enoent)
diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py
index 351183f1..41ea4fc9 100644
--- a/cloudinit/config/cc_mcollective.py
+++ b/cloudinit/config/cc_mcollective.py
@@ -56,6 +56,7 @@ import io
 from configobj import ConfigObj
 
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 
 PUBCERT_FILE = "/etc/mcollective/ssl/server-public.pem"
@@ -140,6 +141,6 @@ def handle(name, cfg, cloud, log, _args):
         configure(config=mcollective_cfg['conf'])
 
     # restart mcollective to handle updated config
-    util.subp(['service', 'mcollective', 'restart'], capture=False)
+    subp.subp(['service', 'mcollective', 'restart'], capture=False)
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py
index 85a89cd1..e57d1b1f 100644
--- a/cloudinit/config/cc_mounts.py
+++ b/cloudinit/config/cc_mounts.py
@@ -69,6 +69,7 @@ import os.path
 import re
 
 from cloudinit import type_utils
+from cloudinit import subp
 from cloudinit import util
 
 # Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0
@@ -252,8 +253,8 @@ def create_swapfile(fname: str, size: str) -> None:
                    'count=%s' % size]
 
         try:
-            util.subp(cmd, capture=True)
-        except util.ProcessExecutionError as e:
+            subp.subp(cmd, capture=True)
+        except subp.ProcessExecutionError as e:
             LOG.warning(errmsg, fname, size, method, e)
             util.del_file(fname)
 
@@ -267,15 +268,15 @@ def create_swapfile(fname: str, size: str) -> None:
     else:
         try:
             create_swap(fname, size, "fallocate")
-        except util.ProcessExecutionError as e:
+        except subp.ProcessExecutionError as e:
             LOG.warning(errmsg, fname, size, "dd", e)
             LOG.warning("Will attempt with dd.")
             create_swap(fname, size, "dd")
 
     util.chmod(fname, 0o600)
     try:
-        util.subp(['mkswap', fname])
-    except util.ProcessExecutionError:
+        subp.subp(['mkswap', fname])
+    except subp.ProcessExecutionError:
         util.del_file(fname)
         raise
 
@@ -538,9 +539,9 @@ def handle(_name, cfg, cloud, log, _args):
     for cmd in activate_cmds:
         fmt = "Activate mounts: %s:" + ' '.join(cmd)
         try:
-            util.subp(cmd)
+            subp.subp(cmd)
             log.debug(fmt, "PASS")
-        except util.ProcessExecutionError:
+        except subp.ProcessExecutionError:
             log.warning(fmt, "FAIL")
             util.logexc(log, fmt, "FAIL")
 
diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py
index 3b2c2020..7d3f73ff 100644
--- a/cloudinit/config/cc_ntp.py
+++ b/cloudinit/config/cc_ntp.py
@@ -14,6 +14,7 @@ from cloudinit import log as logging
 from cloudinit import temp_utils
 from cloudinit import templater
 from cloudinit import type_utils
+from cloudinit import subp
 from cloudinit import util
 from cloudinit.config.schema import get_schema_doc, validate_cloudconfig_schema
 from cloudinit.settings import PER_INSTANCE
@@ -307,7 +308,7 @@ def select_ntp_client(ntp_client, distro):
     if distro_ntp_client == "auto":
         for client in distro.preferred_ntp_clients:
             cfg = distro_cfg.get(client)
-            if util.which(cfg.get('check_exe')):
+            if subp.which(cfg.get('check_exe')):
                 LOG.debug('Selected NTP client "%s", already installed',
                           client)
                 clientcfg = cfg
@@ -336,7 +337,7 @@ def install_ntp_client(install_func, packages=None, check_exe="ntpd"):
     @param check_exe: string.  The name of a binary that indicates the package
     the specified package is already installed.
     """
-    if util.which(check_exe):
+    if subp.which(check_exe):
         return
     if packages is None:
         packages = ['ntp']
@@ -431,7 +432,7 @@ def reload_ntp(service, systemd=False):
         cmd = ['systemctl', 'reload-or-restart', service]
     else:
         cmd = ['service', service, 'restart']
-    util.subp(cmd, capture=True)
+    subp.subp(cmd, capture=True)
 
 
 def supplemental_schema_validation(ntp_config):
@@ -543,7 +544,7 @@ def handle(name, cfg, cloud, log, _args):
     try:
         reload_ntp(ntp_client_config['service_name'],
                    systemd=cloud.distro.uses_systemd())
-    except util.ProcessExecutionError as e:
+    except subp.ProcessExecutionError as e:
         LOG.exception("Failed to reload/start ntp service: %s", e)
         raise
 
diff --git a/cloudinit/config/cc_package_update_upgrade_install.py b/cloudinit/config/cc_package_update_upgrade_install.py
index 86afffef..036baf85 100644
--- a/cloudinit/config/cc_package_update_upgrade_install.py
+++ b/cloudinit/config/cc_package_update_upgrade_install.py
@@ -43,6 +43,7 @@ import os
 import time
 
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 
 REBOOT_FILE = "/var/run/reboot-required"
@@ -57,7 +58,7 @@ def _multi_cfg_bool_get(cfg, *keys):
 
 
 def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2):
-    util.subp(REBOOT_CMD)
+    subp.subp(REBOOT_CMD)
     start = time.time()
     wait_time = initial_sleep
     for _i in range(0, wait_attempts):
diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py
index 3e81a3c7..41ffb46c 100644
--- a/cloudinit/config/cc_power_state_change.py
+++ b/cloudinit/config/cc_power_state_change.py
@@ -56,6 +56,7 @@ import subprocess
 import time
 
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import subp
 from cloudinit import util
 
 frequency = PER_INSTANCE
@@ -71,7 +72,7 @@ def givecmdline(pid):
         #   PID COMM             ARGS
         #     1 init             /bin/init --
         if util.is_FreeBSD():
-            (output, _err) = util.subp(['procstat', '-c', str(pid)])
+            (output, _err) = subp.subp(['procstat', '-c', str(pid)])
             line = output.splitlines()[1]
             m = re.search(r'\d+ (\w|\.|-)+\s+(/\w.+)', line)
             return m.group(2)
diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py
index c01f5b8f..635c73bc 100644
--- a/cloudinit/config/cc_puppet.py
+++ b/cloudinit/config/cc_puppet.py
@@ -83,6 +83,7 @@ import yaml
 from io import StringIO
 
 from cloudinit import helpers
+from cloudinit import subp
 from cloudinit import util
 
 PUPPET_CONF_PATH = '/etc/puppet/puppet.conf'
@@ -105,14 +106,14 @@ class PuppetConstants(object):
 def _autostart_puppet(log):
     # Set puppet to automatically start
     if os.path.exists('/etc/default/puppet'):
-        util.subp(['sed', '-i',
+        subp.subp(['sed', '-i',
                    '-e', 's/^START=.*/START=yes/',
                    '/etc/default/puppet'], capture=False)
     elif os.path.exists('/bin/systemctl'):
-        util.subp(['/bin/systemctl', 'enable', 'puppet.service'],
+        subp.subp(['/bin/systemctl', 'enable', 'puppet.service'],
                   capture=False)
     elif os.path.exists('/sbin/chkconfig'):
-        util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False)
+        subp.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False)
     else:
         log.warning(("Sorry we do not know how to enable"
                      " puppet services on this system"))
@@ -203,6 +204,6 @@ def handle(name, cfg, cloud, log, _args):
     _autostart_puppet(log)
 
     # Start puppetd
-    util.subp(['service', 'puppet', 'start'], capture=False)
+    subp.subp(['service', 'puppet', 'start'], capture=False)
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py
index 01dfc125..8de4db30 100644
--- a/cloudinit/config/cc_resizefs.py
+++ b/cloudinit/config/cc_resizefs.py
@@ -19,6 +19,7 @@ from textwrap import dedent
 from cloudinit.config.schema import (
     get_schema_doc, validate_cloudconfig_schema)
 from cloudinit.settings import PER_ALWAYS
+from cloudinit import subp
 from cloudinit import util
 
 NOBLOCK = "noblock"
@@ -88,11 +89,11 @@ def _resize_zfs(mount_point, devpth):
 
 
 def _get_dumpfs_output(mount_point):
-    return util.subp(['dumpfs', '-m', mount_point])[0]
+    return subp.subp(['dumpfs', '-m', mount_point])[0]
 
 
 def _get_gpart_output(part):
-    return util.subp(['gpart', 'show', part])[0]
+    return subp.subp(['gpart', 'show', part])[0]
 
 
 def _can_skip_resize_ufs(mount_point, devpth):
@@ -306,8 +307,8 @@ def handle(name, cfg, _cloud, log, args):
 
 def do_resize(resize_cmd, log):
     try:
-        util.subp(resize_cmd)
-    except util.ProcessExecutionError:
+        subp.subp(resize_cmd)
+    except subp.ProcessExecutionError:
         util.logexc(log, "Failed to resize filesystem (cmd=%s)", resize_cmd)
         raise
     # TODO(harlowja): Should we add a fsck check after this to make
diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py
index 28c79b83..28d62e9d 100644
--- a/cloudinit/config/cc_rh_subscription.py
+++ b/cloudinit/config/cc_rh_subscription.py
@@ -39,6 +39,7 @@ Subscription`` example config.
 """
 
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -173,7 +174,7 @@ class SubscriptionManager(object):
 
         try:
             _sub_man_cli(cmd)
-        except util.ProcessExecutionError:
+        except subp.ProcessExecutionError:
             return False
 
         return True
@@ -200,7 +201,7 @@ class SubscriptionManager(object):
 
             try:
                 return_out = _sub_man_cli(cmd, logstring_val=True)[0]
-            except util.ProcessExecutionError as e:
+            except subp.ProcessExecutionError as e:
                 if e.stdout == "":
                     self.log_warn("Registration failed due "
                                   "to: {0}".format(e.stderr))
@@ -223,7 +224,7 @@ class SubscriptionManager(object):
             # Attempting to register the system only
             try:
                 return_out = _sub_man_cli(cmd, logstring_val=True)[0]
-            except util.ProcessExecutionError as e:
+            except subp.ProcessExecutionError as e:
                 if e.stdout == "":
                     self.log_warn("Registration failed due "
                                   "to: {0}".format(e.stderr))
@@ -246,7 +247,7 @@ class SubscriptionManager(object):
 
         try:
             return_out = _sub_man_cli(cmd)[0]
-        except util.ProcessExecutionError as e:
+        except subp.ProcessExecutionError as e:
             if e.stdout.rstrip() != '':
                 for line in e.stdout.split("\n"):
                     if line != '':
@@ -264,7 +265,7 @@ class SubscriptionManager(object):
         cmd = ['attach', '--auto']
         try:
             return_out = _sub_man_cli(cmd)[0]
-        except util.ProcessExecutionError as e:
+        except subp.ProcessExecutionError as e:
             self.log_warn("Auto-attach failed with: {0}".format(e))
             return False
         for line in return_out.split("\n"):
@@ -341,7 +342,7 @@ class SubscriptionManager(object):
                                "system: %s", (", ".join(pool_list))
                                .replace('--pool=', ''))
                 return True
-            except util.ProcessExecutionError as e:
+            except subp.ProcessExecutionError as e:
                 self.log_warn("Unable to attach pool {0} "
                               "due to {1}".format(pool, e))
                 return False
@@ -414,7 +415,7 @@ class SubscriptionManager(object):
 
         try:
             _sub_man_cli(cmd)
-        except util.ProcessExecutionError as e:
+        except subp.ProcessExecutionError as e:
             self.log_warn("Unable to alter repos due to {0}".format(e))
             return False
 
@@ -432,11 +433,11 @@ class SubscriptionManager(object):
 
 def _sub_man_cli(cmd, logstring_val=False):
     '''
-    Uses the prefered cloud-init subprocess def of util.subp
+    Uses the prefered cloud-init subprocess def of subp.subp
     and runs subscription-manager.  Breaking this to a
     separate function for later use in mocking and unittests
     '''
-    return util.subp(['subscription-manager'] + cmd,
+    return subp.subp(['subscription-manager'] + cmd,
                      logstring=logstring_val)
 
 
diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py
index 5df0137d..1354885a 100644
--- a/cloudinit/config/cc_rsyslog.py
+++ b/cloudinit/config/cc_rsyslog.py
@@ -182,6 +182,7 @@ import os
 import re
 
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 
 DEF_FILENAME = "20-cloud-config.conf"
@@ -215,7 +216,7 @@ def reload_syslog(command=DEF_RELOAD, systemd=False):
             cmd = ['service', service, 'restart']
     else:
         cmd = command
-    util.subp(cmd, capture=True)
+    subp.subp(cmd, capture=True)
 
 
 def load_config(cfg):
@@ -429,7 +430,7 @@ def handle(name, cfg, cloud, log, _args):
         restarted = reload_syslog(
             command=mycfg[KEYNAME_RELOAD],
             systemd=cloud.distro.uses_systemd()),
-    except util.ProcessExecutionError as e:
+    except subp.ProcessExecutionError as e:
         restarted = False
         log.warning("Failed to reload syslog", e)
 
diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py
index 5dd8de37..b61876aa 100644
--- a/cloudinit/config/cc_salt_minion.py
+++ b/cloudinit/config/cc_salt_minion.py
@@ -45,7 +45,7 @@ specify them with ``pkg_name``, ``service_name`` and ``config_dir``.
 
 import os
 
-from cloudinit import safeyaml, util
+from cloudinit import safeyaml, subp, util
 from cloudinit.distros import rhel_util
 
 
@@ -130,6 +130,6 @@ def handle(name, cfg, cloud, log, _args):
 
     # restart salt-minion. 'service' will start even if not started. if it
     # was started, it needs to be restarted for config change.
-    util.subp(['service', const.srv_name, 'restart'], capture=False)
+    subp.subp(['service', const.srv_name, 'restart'], capture=False)
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py
index b65f3ed9..4fb9b44e 100644
--- a/cloudinit/config/cc_seed_random.py
+++ b/cloudinit/config/cc_seed_random.py
@@ -65,6 +65,7 @@ from io import BytesIO
 
 from cloudinit import log as logging
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import subp
 from cloudinit import util
 
 frequency = PER_INSTANCE
@@ -92,14 +93,14 @@ def handle_random_seed_command(command, required, env=None):
         return
 
     cmd = command[0]
-    if not util.which(cmd):
+    if not subp.which(cmd):
         if required:
             raise ValueError(
                 "command '{cmd}' not found but required=true".format(cmd=cmd))
         else:
             LOG.debug("command '%s' not found for seed_command", cmd)
             return
-    util.subp(command, env=env, capture=False)
+    subp.subp(command, env=env, capture=False)
 
 
 def handle(name, cfg, cloud, log, _args):
diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py
index 7b7aa885..d6b5682d 100755
--- a/cloudinit/config/cc_set_passwords.py
+++ b/cloudinit/config/cc_set_passwords.py
@@ -83,6 +83,7 @@ import sys
 from cloudinit.distros import ug_util
 from cloudinit import log as logging
 from cloudinit.ssh_util import update_ssh_config
+from cloudinit import subp
 from cloudinit import util
 
 from string import ascii_letters, digits
@@ -128,7 +129,7 @@ def handle_ssh_pwauth(pw_auth, service_cmd=None, service_name="ssh"):
         cmd = list(service_cmd) + ["restart", service_name]
     else:
         cmd = list(service_cmd) + [service_name, "restart"]
-    util.subp(cmd)
+    subp.subp(cmd)
     LOG.debug("Restarted the SSH daemon.")
 
 
@@ -247,6 +248,6 @@ def chpasswd(distro, plist_in, hashed=False):
             distro.set_passwd(u, p, hashed=hashed)
     else:
         cmd = ['chpasswd'] + (['-e'] if hashed else [])
-        util.subp(cmd, plist_in)
+        subp.subp(cmd, plist_in)
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py
index 8178562e..20ed7d2f 100644
--- a/cloudinit/config/cc_snap.py
+++ b/cloudinit/config/cc_snap.py
@@ -12,6 +12,7 @@ from cloudinit.config.schema import (
     get_schema_doc, validate_cloudconfig_schema)
 from cloudinit.settings import PER_INSTANCE
 from cloudinit.subp import prepend_base_command
+from cloudinit import subp
 from cloudinit import util
 
 
@@ -175,7 +176,7 @@ def add_assertions(assertions):
         LOG.debug('Snap acking: %s', asrt.split('\n')[0:2])
 
     util.write_file(ASSERTIONS_FILE, combined.encode('utf-8'))
-    util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True)
+    subp.subp(snap_cmd + [ASSERTIONS_FILE], capture=True)
 
 
 def run_commands(commands):
@@ -204,8 +205,8 @@ def run_commands(commands):
     for command in fixed_snap_commands:
         shell = isinstance(command, str)
         try:
-            util.subp(command, shell=shell, status_cb=sys.stderr.write)
-        except util.ProcessExecutionError as e:
+            subp.subp(command, shell=shell, status_cb=sys.stderr.write)
+        except subp.ProcessExecutionError as e:
             cmd_failures.append(str(e))
     if cmd_failures:
         msg = 'Failures running snap commands:\n{cmd_failures}'.format(
diff --git a/cloudinit/config/cc_spacewalk.py b/cloudinit/config/cc_spacewalk.py
index 1020e944..95083607 100644
--- a/cloudinit/config/cc_spacewalk.py
+++ b/cloudinit/config/cc_spacewalk.py
@@ -27,7 +27,7 @@ For more information about spacewalk see: https://fedorahosted.org/spacewalk/
        activation_key: <key>
 """
 
-from cloudinit import util
+from cloudinit import subp
 
 
 distros = ['redhat', 'fedora']
@@ -41,9 +41,9 @@ def is_registered():
     # assume we aren't registered; which is sorta ghetto...
     already_registered = False
     try:
-        util.subp(['rhn-profile-sync', '--verbose'], capture=False)
+        subp.subp(['rhn-profile-sync', '--verbose'], capture=False)
         already_registered = True
-    except util.ProcessExecutionError as e:
+    except subp.ProcessExecutionError as e:
         if e.exit_code != 1:
             raise
     return already_registered
@@ -65,7 +65,7 @@ def do_register(server, profile_name,
         cmd.extend(['--sslCACert', str(ca_cert_path)])
     if activation_key:
         cmd.extend(['--activationkey', str(activation_key)])
-    util.subp(cmd, capture=False)
+    subp.subp(cmd, capture=False)
 
 
 def handle(name, cfg, cloud, log, _args):
diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py
index 163cce99..228e5e0d 100755
--- a/cloudinit/config/cc_ssh.py
+++ b/cloudinit/config/cc_ssh.py
@@ -116,6 +116,7 @@ import sys
 
 from cloudinit.distros import ug_util
 from cloudinit import ssh_util
+from cloudinit import subp
 from cloudinit import util
 
 
@@ -164,7 +165,7 @@ def handle(_name, cfg, cloud, log, _args):
             try:
                 # TODO(harlowja): Is this guard needed?
                 with util.SeLinuxGuard("/etc/ssh", recursive=True):
-                    util.subp(cmd, capture=False)
+                    subp.subp(cmd, capture=False)
                 log.debug("Generated a key for %s from %s", pair[0], pair[1])
             except Exception:
                 util.logexc(log, "Failed generated a key for %s from %s",
@@ -186,9 +187,9 @@ def handle(_name, cfg, cloud, log, _args):
             # TODO(harlowja): Is this guard needed?
             with util.SeLinuxGuard("/etc/ssh", recursive=True):
                 try:
-                    out, err = util.subp(cmd, capture=True, env=lang_c)
+                    out, err = subp.subp(cmd, capture=True, env=lang_c)
                     sys.stdout.write(util.decode_binary(out))
-                except util.ProcessExecutionError as e:
+                except subp.ProcessExecutionError as e:
                     err = util.decode_binary(e.stderr).lower()
                     if (e.exit_code == 1 and
                             err.lower().startswith("unknown key")):
diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py
index 63f87298..856e5a9e 100755
--- a/cloudinit/config/cc_ssh_import_id.py
+++ b/cloudinit/config/cc_ssh_import_id.py
@@ -31,6 +31,7 @@ either ``lp:`` for launchpad or ``gh:`` for github to the username.
 """
 
 from cloudinit.distros import ug_util
+from cloudinit import subp
 from cloudinit import util
 import pwd
 
@@ -101,8 +102,8 @@ def import_ssh_ids(ids, user, log):
     log.debug("Importing SSH ids for user %s.", user)
 
     try:
-        util.subp(cmd, capture=False)
-    except util.ProcessExecutionError as exc:
+        subp.subp(cmd, capture=False)
+    except subp.ProcessExecutionError as exc:
         util.logexc(log, "Failed to run command to import %s SSH ids", user)
         raise exc
 
diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py
index 8b6d2a1a..35ded5db 100644
--- a/cloudinit/config/cc_ubuntu_advantage.py
+++ b/cloudinit/config/cc_ubuntu_advantage.py
@@ -8,6 +8,7 @@ from cloudinit.config.schema import (
     get_schema_doc, validate_cloudconfig_schema)
 from cloudinit import log as logging
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import subp
 from cloudinit import util
 
 
@@ -109,8 +110,8 @@ def configure_ua(token=None, enable=None):
     attach_cmd = ['ua', 'attach', token]
     LOG.debug('Attaching to Ubuntu Advantage. %s', ' '.join(attach_cmd))
     try:
-        util.subp(attach_cmd)
-    except util.ProcessExecutionError as e:
+        subp.subp(attach_cmd)
+    except subp.ProcessExecutionError as e:
         msg = 'Failure attaching Ubuntu Advantage:\n{error}'.format(
             error=str(e))
         util.logexc(LOG, msg)
@@ -119,8 +120,8 @@ def configure_ua(token=None, enable=None):
     for service in enable:
         try:
             cmd = ['ua', 'enable', service]
-            util.subp(cmd, capture=True)
-        except util.ProcessExecutionError as e:
+            subp.subp(cmd, capture=True)
+        except subp.ProcessExecutionError as e:
             enable_errors.append((service, e))
     if enable_errors:
         for service, error in enable_errors:
@@ -135,7 +136,7 @@ def configure_ua(token=None, enable=None):
 
 def maybe_install_ua_tools(cloud):
     """Install ubuntu-advantage-tools if not present."""
-    if util.which('ua'):
+    if subp.which('ua'):
         return
     try:
         cloud.distro.update_package_sources()
diff --git a/cloudinit/config/cc_ubuntu_drivers.py b/cloudinit/config/cc_ubuntu_drivers.py
index 297451d6..2d1d2b32 100644
--- a/cloudinit/config/cc_ubuntu_drivers.py
+++ b/cloudinit/config/cc_ubuntu_drivers.py
@@ -9,6 +9,7 @@ from cloudinit.config.schema import (
     get_schema_doc, validate_cloudconfig_schema)
 from cloudinit import log as logging
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import subp
 from cloudinit import temp_utils
 from cloudinit import type_utils
 from cloudinit import util
@@ -108,7 +109,7 @@ def install_drivers(cfg, pkg_install_func):
         LOG.debug("Not installing NVIDIA drivers. %s=%s", cfgpath, nv_acc)
         return
 
-    if not util.which('ubuntu-drivers'):
+    if not subp.which('ubuntu-drivers'):
         LOG.debug("'ubuntu-drivers' command not available.  "
                   "Installing ubuntu-drivers-common")
         pkg_install_func(['ubuntu-drivers-common'])
@@ -131,7 +132,7 @@ def install_drivers(cfg, pkg_install_func):
             debconf_script,
             util.encode_text(NVIDIA_DRIVER_LATELINK_DEBCONF_SCRIPT),
             mode=0o755)
-        util.subp([debconf_script, debconf_file])
+        subp.subp([debconf_script, debconf_file])
     except Exception as e:
         util.logexc(
             LOG, "Failed to register NVIDIA debconf template: %s", str(e))
@@ -141,8 +142,8 @@ def install_drivers(cfg, pkg_install_func):
             util.del_dir(tdir)
 
     try:
-        util.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg])
-    except util.ProcessExecutionError as exc:
+        subp.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg])
+    except subp.ProcessExecutionError as exc:
         if OLD_UBUNTU_DRIVERS_STDERR_NEEDLE in exc.stderr:
             LOG.warning('the available version of ubuntu-drivers is'
                         ' too old to perform requested driver installation')
diff --git a/cloudinit/config/tests/test_disable_ec2_metadata.py b/cloudinit/config/tests/test_disable_ec2_metadata.py
index 823917c7..b00f2083 100644
--- a/cloudinit/config/tests/test_disable_ec2_metadata.py
+++ b/cloudinit/config/tests/test_disable_ec2_metadata.py
@@ -15,8 +15,8 @@ DISABLE_CFG = {'disable_ec2_metadata': 'true'}
 
 class TestEC2MetadataRoute(CiTestCase):
 
-    @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which')
-    @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp')
+    @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which')
+    @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp')
     def test_disable_ifconfig(self, m_subp, m_which):
         """Set the route if ifconfig command is available"""
         m_which.side_effect = lambda x: x if x == 'ifconfig' else None
@@ -25,8 +25,8 @@ class TestEC2MetadataRoute(CiTestCase):
             ['route', 'add', '-host', '169.254.169.254', 'reject'],
             capture=False)
 
-    @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which')
-    @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp')
+    @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which')
+    @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp')
     def test_disable_ip(self, m_subp, m_which):
         """Set the route if ip command is available"""
         m_which.side_effect = lambda x: x if x == 'ip' else None
@@ -35,8 +35,8 @@ class TestEC2MetadataRoute(CiTestCase):
             ['ip', 'route', 'add', 'prohibit', '169.254.169.254'],
             capture=False)
 
-    @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which')
-    @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp')
+    @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which')
+    @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp')
     def test_disable_no_tool(self, m_subp, m_which):
         """Log error when neither route nor ip commands are available"""
         m_which.return_value = None  # Find neither ifconfig nor ip
diff --git a/cloudinit/config/tests/test_grub_dpkg.py b/cloudinit/config/tests/test_grub_dpkg.py
index 01efa330..99c05bb5 100644
--- a/cloudinit/config/tests/test_grub_dpkg.py
+++ b/cloudinit/config/tests/test_grub_dpkg.py
@@ -4,7 +4,7 @@ import pytest
 
 from unittest import mock
 from logging import Logger
-from cloudinit.util import ProcessExecutionError
+from cloudinit.subp import ProcessExecutionError
 from cloudinit.config.cc_grub_dpkg import fetch_idevs, handle
 
 
@@ -79,7 +79,7 @@ class TestFetchIdevs:
     )
     @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc")
     @mock.patch("cloudinit.config.cc_grub_dpkg.os.path.exists")
-    @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.subp.subp")
     def test_fetch_idevs(self, m_subp, m_exists, m_logexc, grub_output,
                          path_exists, expected_log_call, udevadm_output,
                          expected_idevs):
@@ -158,7 +158,7 @@ class TestHandle:
     @mock.patch("cloudinit.config.cc_grub_dpkg.fetch_idevs")
     @mock.patch("cloudinit.config.cc_grub_dpkg.util.get_cfg_option_str")
     @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc")
-    @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.subp.subp")
     def test_handle(self, m_subp, m_logexc, m_get_cfg_str, m_fetch_idevs,
                     cfg_idevs, cfg_idevs_empty, fetch_idevs_output,
                     expected_log_output):
diff --git a/cloudinit/config/tests/test_mounts.py b/cloudinit/config/tests/test_mounts.py
index 80b54d0f..764a33e3 100644
--- a/cloudinit/config/tests/test_mounts.py
+++ b/cloudinit/config/tests/test_mounts.py
@@ -13,12 +13,12 @@ class TestCreateSwapfile:
 
     @pytest.mark.parametrize('fstype', ('xfs', 'btrfs', 'ext4', 'other'))
     @mock.patch(M_PATH + 'util.get_mount_info')
-    @mock.patch(M_PATH + 'util.subp')
+    @mock.patch(M_PATH + 'subp.subp')
     def test_happy_path(self, m_subp, m_get_mount_info, fstype, tmpdir):
         swap_file = tmpdir.join("swap-file")
         fname = str(swap_file)
 
-        # Some of the calls to util.subp should create the swap file; this
+        # Some of the calls to subp.subp should create the swap file; this
         # roughly approximates that
         m_subp.side_effect = lambda *args, **kwargs: swap_file.write('')
 
diff --git a/cloudinit/config/tests/test_set_passwords.py b/cloudinit/config/tests/test_set_passwords.py
index 2732bd60..daa1ef51 100644
--- a/cloudinit/config/tests/test_set_passwords.py
+++ b/cloudinit/config/tests/test_set_passwords.py
@@ -14,7 +14,7 @@ class TestHandleSshPwauth(CiTestCase):
 
     with_logs = True
 
-    @mock.patch(MODPATH + "util.subp")
+    @mock.patch(MODPATH + "subp.subp")
     def test_unknown_value_logs_warning(self, m_subp):
         setpass.handle_ssh_pwauth("floo")
         self.assertIn("Unrecognized value: ssh_pwauth=floo",
@@ -22,7 +22,7 @@ class TestHandleSshPwauth(CiTestCase):
         m_subp.assert_not_called()
 
     @mock.patch(MODPATH + "update_ssh_config", return_value=True)
-    @mock.patch(MODPATH + "util.subp")
+    @mock.patch(MODPATH + "subp.subp")
     def test_systemctl_as_service_cmd(self, m_subp, m_update_ssh_config):
         """If systemctl in service cmd: systemctl restart name."""
         setpass.handle_ssh_pwauth(
@@ -31,7 +31,7 @@ class TestHandleSshPwauth(CiTestCase):
                          m_subp.call_args)
 
     @mock.patch(MODPATH + "update_ssh_config", return_value=True)
-    @mock.patch(MODPATH + "util.subp")
+    @mock.patch(MODPATH + "subp.subp")
     def test_service_as_service_cmd(self, m_subp, m_update_ssh_config):
         """If systemctl in service cmd: systemctl restart name."""
         setpass.handle_ssh_pwauth(
@@ -40,7 +40,7 @@ class TestHandleSshPwauth(CiTestCase):
                          m_subp.call_args)
 
     @mock.patch(MODPATH + "update_ssh_config", return_value=False)
-    @mock.patch(MODPATH + "util.subp")
+    @mock.patch(MODPATH + "subp.subp")
     def test_not_restarted_if_not_updated(self, m_subp, m_update_ssh_config):
         """If config is not updated, then no system restart should be done."""
         setpass.handle_ssh_pwauth(True)
@@ -48,7 +48,7 @@ class TestHandleSshPwauth(CiTestCase):
         self.assertIn("No need to restart SSH", self.logs.getvalue())
 
     @mock.patch(MODPATH + "update_ssh_config", return_value=True)
-    @mock.patch(MODPATH + "util.subp")
+    @mock.patch(MODPATH + "subp.subp")
     def test_unchanged_does_nothing(self, m_subp, m_update_ssh_config):
         """If 'unchanged', then no updates to config and no restart."""
         setpass.handle_ssh_pwauth(
@@ -56,7 +56,7 @@ class TestHandleSshPwauth(CiTestCase):
         m_update_ssh_config.assert_not_called()
         m_subp.assert_not_called()
 
-    @mock.patch(MODPATH + "util.subp")
+    @mock.patch(MODPATH + "subp.subp")
     def test_valid_change_values(self, m_subp):
         """If value is a valid changen value, then update should be called."""
         upname = MODPATH + "update_ssh_config"
@@ -88,7 +88,7 @@ class TestSetPasswordsHandle(CiTestCase):
             'ssh_pwauth=None\n',
             self.logs.getvalue())
 
-    @mock.patch(MODPATH + "util.subp")
+    @mock.patch(MODPATH + "subp.subp")
     def test_handle_on_chpasswd_list_parses_common_hashes(self, m_subp):
         """handle parses command password hashes."""
         cloud = self.tmp_cloud(distro='ubuntu')
@@ -98,7 +98,7 @@ class TestSetPasswordsHandle(CiTestCase):
             'ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoakMMC7dR52q'
             'SDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXazGGx3oo1']
         cfg = {'chpasswd': {'list': valid_hashed_pwds}}
-        with mock.patch(MODPATH + 'util.subp') as m_subp:
+        with mock.patch(MODPATH + 'subp.subp') as m_subp:
             setpass.handle(
                 'IGNORED', cfg=cfg, cloud=cloud, log=self.logger, args=[])
         self.assertIn(
@@ -113,7 +113,7 @@ class TestSetPasswordsHandle(CiTestCase):
             m_subp.call_args_list)
 
     @mock.patch(MODPATH + "util.is_BSD")
-    @mock.patch(MODPATH + "util.subp")
+    @mock.patch(MODPATH + "subp.subp")
     def test_bsd_calls_custom_pw_cmds_to_set_and_expire_passwords(
             self, m_subp, m_is_bsd):
         """BSD don't use chpasswd"""
@@ -130,7 +130,7 @@ class TestSetPasswordsHandle(CiTestCase):
             m_subp.call_args_list)
 
     @mock.patch(MODPATH + "util.is_BSD")
-    @mock.patch(MODPATH + "util.subp")
+    @mock.patch(MODPATH + "subp.subp")
     def test_handle_on_chpasswd_list_creates_random_passwords(self, m_subp,
                                                               m_is_bsd):
         """handle parses command set random passwords."""
@@ -140,7 +140,7 @@ class TestSetPasswordsHandle(CiTestCase):
             'root:R',
             'ubuntu:RANDOM']
         cfg = {'chpasswd': {'expire': 'false', 'list': valid_random_pwds}}
-        with mock.patch(MODPATH + 'util.subp') as m_subp:
+        with mock.patch(MODPATH + 'subp.subp') as m_subp:
             setpass.handle(
                 'IGNORED', cfg=cfg, cloud=cloud, log=self.logger, args=[])
         self.assertIn(
diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py
index 95270fa0..6d4c014a 100644
--- a/cloudinit/config/tests/test_snap.py
+++ b/cloudinit/config/tests/test_snap.py
@@ -92,7 +92,7 @@ class TestAddAssertions(CiTestCase):
         super(TestAddAssertions, self).setUp()
         self.tmp = self.tmp_dir()
 
-    @mock.patch('cloudinit.config.cc_snap.util.subp')
+    @mock.patch('cloudinit.config.cc_snap.subp.subp')
     def test_add_assertions_on_empty_list(self, m_subp):
         """When provided with an empty list, add_assertions does nothing."""
         add_assertions([])
@@ -107,7 +107,7 @@ class TestAddAssertions(CiTestCase):
             "assertion parameter was not a list or dict: I'm Not Valid",
             str(context_manager.exception))
 
-    @mock.patch('cloudinit.config.cc_snap.util.subp')
+    @mock.patch('cloudinit.config.cc_snap.subp.subp')
     def test_add_assertions_adds_assertions_as_list(self, m_subp):
         """When provided with a list, add_assertions adds all assertions."""
         self.assertEqual(
@@ -130,7 +130,7 @@ class TestAddAssertions(CiTestCase):
         self.assertEqual(
             util.load_file(compare_file), util.load_file(assert_file))
 
-    @mock.patch('cloudinit.config.cc_snap.util.subp')
+    @mock.patch('cloudinit.config.cc_snap.subp.subp')
     def test_add_assertions_adds_assertions_as_dict(self, m_subp):
         """When provided with a dict, add_assertions adds all assertions."""
         self.assertEqual(
@@ -168,7 +168,7 @@ class TestRunCommands(CiTestCase):
         super(TestRunCommands, self).setUp()
         self.tmp = self.tmp_dir()
 
-    @mock.patch('cloudinit.config.cc_snap.util.subp')
+    @mock.patch('cloudinit.config.cc_snap.subp.subp')
     def test_run_commands_on_empty_list(self, m_subp):
         """When provided with an empty list, run_commands does nothing."""
         run_commands([])
@@ -477,7 +477,7 @@ class TestHandle(CiTestCase):
 
         self.assertEqual('HI\nMOM\n', util.load_file(outfile))
 
-    @mock.patch('cloudinit.config.cc_snap.util.subp')
+    @mock.patch('cloudinit.config.cc_snap.subp.subp')
     def test_handle_adds_assertions(self, m_subp):
         """Any configured snap assertions are provided to add_assertions."""
         assert_file = self.tmp_path('snapd.assertions', dir=self.tmp)
@@ -493,7 +493,7 @@ class TestHandle(CiTestCase):
         self.assertEqual(
             util.load_file(compare_file), util.load_file(assert_file))
 
-    @mock.patch('cloudinit.config.cc_snap.util.subp')
+    @mock.patch('cloudinit.config.cc_snap.subp.subp')
     @skipUnlessJsonSchema()
     def test_handle_validates_schema(self, m_subp):
         """Any provided configuration is runs validate_cloudconfig_schema."""
diff --git a/cloudinit/config/tests/test_ubuntu_advantage.py b/cloudinit/config/tests/test_ubuntu_advantage.py
index 8c4161ef..db7fb726 100644
--- a/cloudinit/config/tests/test_ubuntu_advantage.py
+++ b/cloudinit/config/tests/test_ubuntu_advantage.py
@@ -3,7 +3,7 @@
 from cloudinit.config.cc_ubuntu_advantage import (
     configure_ua, handle, maybe_install_ua_tools, schema)
 from cloudinit.config.schema import validate_cloudconfig_schema
-from cloudinit import util
+from cloudinit import subp
 from cloudinit.tests.helpers import (
     CiTestCase, mock, SchemaTestCaseMixin, skipUnlessJsonSchema)
 
@@ -26,10 +26,10 @@ class TestConfigureUA(CiTestCase):
         super(TestConfigureUA, self).setUp()
         self.tmp = self.tmp_dir()
 
-    @mock.patch('%s.util.subp' % MPATH)
+    @mock.patch('%s.subp.subp' % MPATH)
     def test_configure_ua_attach_error(self, m_subp):
         """Errors from ua attach command are raised."""
-        m_subp.side_effect = util.ProcessExecutionError(
+        m_subp.side_effect = subp.ProcessExecutionError(
             'Invalid token SomeToken')
         with self.assertRaises(RuntimeError) as context_manager:
             configure_ua(token='SomeToken')
@@ -39,7 +39,7 @@ class TestConfigureUA(CiTestCase):
             'Stdout: Invalid token SomeToken\nStderr: -',
             str(context_manager.exception))
 
-    @mock.patch('%s.util.subp' % MPATH)
+    @mock.patch('%s.subp.subp' % MPATH)
     def test_configure_ua_attach_with_token(self, m_subp):
         """When token is provided, attach the machine to ua using the token."""
         configure_ua(token='SomeToken')
@@ -48,7 +48,7 @@ class TestConfigureUA(CiTestCase):
             'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n',
             self.logs.getvalue())
 
-    @mock.patch('%s.util.subp' % MPATH)
+    @mock.patch('%s.subp.subp' % MPATH)
     def test_configure_ua_attach_on_service_error(self, m_subp):
         """all services should be enabled and then any failures raised"""
 
@@ -56,7 +56,7 @@ class TestConfigureUA(CiTestCase):
             fail_cmds = [['ua', 'enable', svc] for svc in ['esm', 'cc']]
             if cmd in fail_cmds and capture:
                 svc = cmd[-1]
-                raise util.ProcessExecutionError(
+                raise subp.ProcessExecutionError(
                     'Invalid {} credentials'.format(svc.upper()))
 
         m_subp.side_effect = fake_subp
@@ -83,7 +83,7 @@ class TestConfigureUA(CiTestCase):
             'Failure enabling Ubuntu Advantage service(s): "esm", "cc"',
             str(context_manager.exception))
 
-    @mock.patch('%s.util.subp' % MPATH)
+    @mock.patch('%s.subp.subp' % MPATH)
     def test_configure_ua_attach_with_empty_services(self, m_subp):
         """When services is an empty list, do not auto-enable attach."""
         configure_ua(token='SomeToken', enable=[])
@@ -92,7 +92,7 @@ class TestConfigureUA(CiTestCase):
             'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n',
             self.logs.getvalue())
 
-    @mock.patch('%s.util.subp' % MPATH)
+    @mock.patch('%s.subp.subp' % MPATH)
     def test_configure_ua_attach_with_specific_services(self, m_subp):
         """When services a list, only enable specific services."""
         configure_ua(token='SomeToken', enable=['fips'])
@@ -105,7 +105,7 @@ class TestConfigureUA(CiTestCase):
             self.logs.getvalue())
 
     @mock.patch('%s.maybe_install_ua_tools' % MPATH, mock.MagicMock())
-    @mock.patch('%s.util.subp' % MPATH)
+    @mock.patch('%s.subp.subp' % MPATH)
     def test_configure_ua_attach_with_string_services(self, m_subp):
         """When services a string, treat as singleton list and warn"""
         configure_ua(token='SomeToken', enable='fips')
@@ -119,7 +119,7 @@ class TestConfigureUA(CiTestCase):
             'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n',
             self.logs.getvalue())
 
-    @mock.patch('%s.util.subp' % MPATH)
+    @mock.patch('%s.subp.subp' % MPATH)
     def test_configure_ua_attach_with_weird_services(self, m_subp):
         """When services not string or list, warn but still attach"""
         configure_ua(token='SomeToken', enable={'deffo': 'wont work'})
@@ -285,7 +285,7 @@ class TestMaybeInstallUATools(CiTestCase):
         super(TestMaybeInstallUATools, self).setUp()
         self.tmp = self.tmp_dir()
 
-    @mock.patch('%s.util.which' % MPATH)
+    @mock.patch('%s.subp.which' % MPATH)
     def test_maybe_install_ua_tools_noop_when_ua_tools_present(self, m_which):
         """Do nothing if ubuntu-advantage-tools already exists."""
         m_which.return_value = '/usr/bin/ua'  # already installed
@@ -294,7 +294,7 @@ class TestMaybeInstallUATools(CiTestCase):
             'Some apt error')
         maybe_install_ua_tools(cloud=FakeCloud(distro))  # No RuntimeError
 
-    @mock.patch('%s.util.which' % MPATH)
+    @mock.patch('%s.subp.which' % MPATH)
     def test_maybe_install_ua_tools_raises_update_errors(self, m_which):
         """maybe_install_ua_tools logs and raises apt update errors."""
         m_which.return_value = None
@@ -306,7 +306,7 @@ class TestMaybeInstallUATools(CiTestCase):
         self.assertEqual('Some apt error', str(context_manager.exception))
         self.assertIn('Package update failed\nTraceback', self.logs.getvalue())
 
-    @mock.patch('%s.util.which' % MPATH)
+    @mock.patch('%s.subp.which' % MPATH)
     def test_maybe_install_ua_raises_install_errors(self, m_which):
         """maybe_install_ua_tools logs and raises package install errors."""
         m_which.return_value = None
@@ -320,7 +320,7 @@ class TestMaybeInstallUATools(CiTestCase):
         self.assertIn(
             'Failed to install ubuntu-advantage-tools\n', self.logs.getvalue())
 
-    @mock.patch('%s.util.which' % MPATH)
+    @mock.patch('%s.subp.which' % MPATH)
     def test_maybe_install_ua_tools_happy_path(self, m_which):
         """maybe_install_ua_tools installs ubuntu-advantage-tools."""
         m_which.return_value = None
diff --git a/cloudinit/config/tests/test_ubuntu_drivers.py b/cloudinit/config/tests/test_ubuntu_drivers.py
index 0aec1265..504ba356 100644
--- a/cloudinit/config/tests/test_ubuntu_drivers.py
+++ b/cloudinit/config/tests/test_ubuntu_drivers.py
@@ -7,7 +7,7 @@ from cloudinit.tests.helpers import CiTestCase, skipUnlessJsonSchema, mock
 from cloudinit.config.schema import (
     SchemaValidationError, validate_cloudconfig_schema)
 from cloudinit.config import cc_ubuntu_drivers as drivers
-from cloudinit.util import ProcessExecutionError
+from cloudinit.subp import ProcessExecutionError
 
 MPATH = "cloudinit.config.cc_ubuntu_drivers."
 M_TMP_PATH = MPATH + "temp_utils.mkdtemp"
@@ -53,8 +53,8 @@ class TestUbuntuDrivers(CiTestCase):
                 schema=drivers.schema, strict=True)
 
     @mock.patch(M_TMP_PATH)
-    @mock.patch(MPATH + "util.subp", return_value=('', ''))
-    @mock.patch(MPATH + "util.which", return_value=False)
+    @mock.patch(MPATH + "subp.subp", return_value=('', ''))
+    @mock.patch(MPATH + "subp.which", return_value=False)
     def _assert_happy_path_taken(
             self, config, m_which, m_subp, m_tmp):
         """Positive path test through handle. Package should be installed."""
@@ -80,8 +80,8 @@ class TestUbuntuDrivers(CiTestCase):
             self._assert_happy_path_taken(new_config)
 
     @mock.patch(M_TMP_PATH)
-    @mock.patch(MPATH + "util.subp")
-    @mock.patch(MPATH + "util.which", return_value=False)
+    @mock.patch(MPATH + "subp.subp")
+    @mock.patch(MPATH + "subp.which", return_value=False)
     def test_handle_raises_error_if_no_drivers_found(
             self, m_which, m_subp, m_tmp):
         """If ubuntu-drivers doesn't install any drivers, raise an error."""
@@ -109,8 +109,8 @@ class TestUbuntuDrivers(CiTestCase):
         self.assertIn('ubuntu-drivers found no drivers for installation',
                       self.logs.getvalue())
 
-    @mock.patch(MPATH + "util.subp", return_value=('', ''))
-    @mock.patch(MPATH + "util.which", return_value=False)
+    @mock.patch(MPATH + "subp.subp", return_value=('', ''))
+    @mock.patch(MPATH + "subp.which", return_value=False)
     def _assert_inert_with_config(self, config, m_which, m_subp):
         """Helper to reduce repetition when testing negative cases"""
         myCloud = mock.MagicMock()
@@ -154,8 +154,8 @@ class TestUbuntuDrivers(CiTestCase):
         self.assertEqual(0, m_install_drivers.call_count)
 
     @mock.patch(M_TMP_PATH)
-    @mock.patch(MPATH + "util.subp", return_value=('', ''))
-    @mock.patch(MPATH + "util.which", return_value=True)
+    @mock.patch(MPATH + "subp.subp", return_value=('', ''))
+    @mock.patch(MPATH + "subp.which", return_value=True)
     def test_install_drivers_no_install_if_present(
             self, m_which, m_subp, m_tmp):
         """If 'ubuntu-drivers' is present, no package install should occur."""
@@ -181,8 +181,8 @@ class TestUbuntuDrivers(CiTestCase):
         self.assertEqual(0, pkg_install.call_count)
 
     @mock.patch(M_TMP_PATH)
-    @mock.patch(MPATH + "util.subp")
-    @mock.patch(MPATH + "util.which", return_value=False)
+    @mock.patch(MPATH + "subp.subp")
+    @mock.patch(MPATH + "subp.which", return_value=False)
     def test_install_drivers_handles_old_ubuntu_drivers_gracefully(
             self, m_which, m_subp, m_tmp):
         """Older ubuntu-drivers versions should emit message and raise error"""
@@ -219,8 +219,8 @@ class TestUbuntuDriversWithVersion(TestUbuntuDrivers):
     install_gpgpu = ['ubuntu-drivers', 'install', '--gpgpu', 'nvidia:123']
 
     @mock.patch(M_TMP_PATH)
-    @mock.patch(MPATH + "util.subp", return_value=('', ''))
-    @mock.patch(MPATH + "util.which", return_value=False)
+    @mock.patch(MPATH + "subp.subp", return_value=('', ''))
+    @mock.patch(MPATH + "subp.which", return_value=False)
     def test_version_none_uses_latest(self, m_which, m_subp, m_tmp):
         tdir = self.tmp_dir()
         debconf_file = os.path.join(tdir, 'nvidia.template')
diff --git a/cloudinit/conftest.py b/cloudinit/conftest.py
index 37cbbcda..251bca59 100644
--- a/cloudinit/conftest.py
+++ b/cloudinit/conftest.py
@@ -2,32 +2,32 @@ from unittest import mock
 
 import pytest
 
-from cloudinit import util
+from cloudinit import subp
 
 
 @pytest.yield_fixture(autouse=True)
 def disable_subp_usage(request):
     """
-    Across all (pytest) tests, ensure that util.subp is not invoked.
+    Across all (pytest) tests, ensure that subp.subp is not invoked.
 
     Note that this can only catch invocations where the util module is imported
-    and ``util.subp(...)`` is called.  ``from cloudinit.util import subp``
+    and ``subp.subp(...)`` is called.  ``from cloudinit.subp mport subp``
     imports happen before the patching here (or the CiTestCase monkey-patching)
     happens, so are left untouched.
 
-    To allow a particular test method or class to use util.subp you can set the
+    To allow a particular test method or class to use subp.subp you can set the
     parameter passed to this fixture to False using pytest.mark.parametrize::
 
         @pytest.mark.parametrize("disable_subp_usage", [False], indirect=True)
         def test_whoami(self):
-            util.subp(["whoami"])
+            subp.subp(["whoami"])
 
-    To instead allow util.subp usage for a specific command, you can set the
+    To instead allow subp.subp usage for a specific command, you can set the
     parameter passed to this fixture to that command:
 
         @pytest.mark.parametrize("disable_subp_usage", ["bash"], indirect=True)
         def test_bash(self):
-            util.subp(["bash"])
+            subp.subp(["bash"])
 
     To specify multiple commands, set the parameter to a list (note the
     double-layered list: we specify a single parameter that is itself a list):
@@ -35,8 +35,8 @@ def disable_subp_usage(request):
         @pytest.mark.parametrize(
             "disable_subp_usage", ["bash", "whoami"], indirect=True)
         def test_several_things(self):
-            util.subp(["bash"])
-            util.subp(["whoami"])
+            subp.subp(["bash"])
+            subp.subp(["whoami"])
 
     This fixture (roughly) mirrors the functionality of
     CiTestCase.allowed_subp.  N.B. While autouse fixtures do affect non-pytest
@@ -47,11 +47,11 @@ def disable_subp_usage(request):
     if should_disable:
         if not isinstance(should_disable, (list, str)):
             def side_effect(args, *other_args, **kwargs):
-                raise AssertionError("Unexpectedly used util.subp")
+                raise AssertionError("Unexpectedly used subp.subp")
         else:
             # Look this up before our patch is in place, so we have access to
             # the real implementation in side_effect
-            subp = util.subp
+            real_subp = subp.subp
 
             if isinstance(should_disable, str):
                 should_disable = [should_disable]
@@ -60,12 +60,12 @@ def disable_subp_usage(request):
                 cmd = args[0]
                 if cmd not in should_disable:
                     raise AssertionError(
-                        "Unexpectedly used util.subp to call {} (allowed:"
+                        "Unexpectedly used subp.subp to call {} (allowed:"
                         " {})".format(cmd, ",".join(should_disable))
                     )
-                return subp(args, *other_args, **kwargs)
+                return real_subp(args, *other_args, **kwargs)
 
-        with mock.patch('cloudinit.util.subp', autospec=True) as m_subp:
+        with mock.patch('cloudinit.subp.subp', autospec=True) as m_subp:
             m_subp.side_effect = side_effect
             yield
     else:
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 35a10590..016ba64d 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -25,6 +25,7 @@ from cloudinit.net import network_state
 from cloudinit.net import renderers
 from cloudinit import ssh_util
 from cloudinit import type_utils
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.distros.parsers import hosts
@@ -225,8 +226,8 @@ class Distro(metaclass=abc.ABCMeta):
         LOG.debug("Non-persistently setting the system hostname to %s",
                   hostname)
         try:
-            util.subp(['hostname', hostname])
-        except util.ProcessExecutionError:
+            subp.subp(['hostname', hostname])
+        except subp.ProcessExecutionError:
             util.logexc(LOG, "Failed to non-persistently adjust the system "
                         "hostname to %s", hostname)
 
@@ -361,12 +362,12 @@ class Distro(metaclass=abc.ABCMeta):
         LOG.debug("Attempting to run bring up interface %s using command %s",
                   device_name, cmd)
         try:
-            (_out, err) = util.subp(cmd)
+            (_out, err) = subp.subp(cmd)
             if len(err):
                 LOG.warning("Running %s resulted in stderr output: %s",
                             cmd, err)
             return True
-        except util.ProcessExecutionError:
+        except subp.ProcessExecutionError:
             util.logexc(LOG, "Running interface command %s failed", cmd)
             return False
 
@@ -480,7 +481,7 @@ class Distro(metaclass=abc.ABCMeta):
         # Run the command
         LOG.debug("Adding user %s", name)
         try:
-            util.subp(useradd_cmd, logstring=log_useradd_cmd)
+            subp.subp(useradd_cmd, logstring=log_useradd_cmd)
         except Exception as e:
             util.logexc(LOG, "Failed to create user %s", name)
             raise e
@@ -500,7 +501,7 @@ class Distro(metaclass=abc.ABCMeta):
         # Run the command
         LOG.debug("Adding snap user %s", name)
         try:
-            (out, err) = util.subp(create_user_cmd, logstring=create_user_cmd,
+            (out, err) = subp.subp(create_user_cmd, logstring=create_user_cmd,
                                    capture=True)
             LOG.debug("snap create-user returned: %s:%s", out, err)
             jobj = util.load_json(out)
@@ -582,20 +583,20 @@ class Distro(metaclass=abc.ABCMeta):
         # passwd must use short '-l' due to SLES11 lacking long form '--lock'
         lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name])
         try:
-            cmd = next(tool for tool in lock_tools if util.which(tool[0]))
+            cmd = next(tool for tool in lock_tools if subp.which(tool[0]))
         except StopIteration:
             raise RuntimeError((
                 "Unable to lock user account '%s'. No tools available. "
                 "  Tried: %s.") % (name, [c[0] for c in lock_tools]))
         try:
-            util.subp(cmd)
+            subp.subp(cmd)
         except Exception as e:
             util.logexc(LOG, 'Failed to disable password for user %s', name)
             raise e
 
     def expire_passwd(self, user):
         try:
-            util.subp(['passwd', '--expire', user])
+            subp.subp(['passwd', '--expire', user])
         except Exception as e:
             util.logexc(LOG, "Failed to set 'expire' for %s", user)
             raise e
@@ -611,7 +612,7 @@ class Distro(metaclass=abc.ABCMeta):
             cmd.append('-e')
 
         try:
-            util.subp(cmd, pass_string, logstring="chpasswd for %s" % user)
+            subp.subp(cmd, pass_string, logstring="chpasswd for %s" % user)
         except Exception as e:
             util.logexc(LOG, "Failed to set password for %s", user)
             raise e
@@ -708,7 +709,7 @@ class Distro(metaclass=abc.ABCMeta):
             LOG.warning("Skipping creation of existing group '%s'", name)
         else:
             try:
-                util.subp(group_add_cmd)
+                subp.subp(group_add_cmd)
                 LOG.info("Created new group %s", name)
             except Exception:
                 util.logexc(LOG, "Failed to create group %s", name)
@@ -721,7 +722,7 @@ class Distro(metaclass=abc.ABCMeta):
                                 "; user does not exist.", member, name)
                     continue
 
-                util.subp(['usermod', '-a', '-G', name, member])
+                subp.subp(['usermod', '-a', '-G', name, member])
                 LOG.info("Added user '%s' to group '%s'", member, name)
 
 
diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py
index 9f89c5f9..038aa9ac 100644
--- a/cloudinit/distros/arch.py
+++ b/cloudinit/distros/arch.py
@@ -8,6 +8,7 @@ from cloudinit import distros
 from cloudinit import helpers
 from cloudinit import log as logging
 from cloudinit import util
+from cloudinit import subp
 
 from cloudinit.distros import net_util
 from cloudinit.distros.parsers.hostname import HostnameConf
@@ -44,7 +45,7 @@ class Distro(distros.Distro):
     def apply_locale(self, locale, out_fn=None):
         if not out_fn:
             out_fn = self.locale_conf_fn
-        util.subp(['locale-gen', '-G', locale], capture=False)
+        subp.subp(['locale-gen', '-G', locale], capture=False)
         # "" provides trailing newline during join
         lines = [
             util.make_header(),
@@ -76,11 +77,11 @@ class Distro(distros.Distro):
     def _enable_interface(self, device_name):
         cmd = ['netctl', 'reenable', device_name]
         try:
-            (_out, err) = util.subp(cmd)
+            (_out, err) = subp.subp(cmd)
             if len(err):
                 LOG.warning("Running %s resulted in stderr output: %s",
                             cmd, err)
-        except util.ProcessExecutionError:
+        except subp.ProcessExecutionError:
             util.logexc(LOG, "Running interface command %s failed", cmd)
 
     def _bring_up_interface(self, device_name):
@@ -88,12 +89,12 @@ class Distro(distros.Distro):
         LOG.debug("Attempting to run bring up interface %s using command %s",
                   device_name, cmd)
         try:
-            (_out, err) = util.subp(cmd)
+            (_out, err) = subp.subp(cmd)
             if len(err):
                 LOG.warning("Running %s resulted in stderr output: %s",
                             cmd, err)
             return True
-        except util.ProcessExecutionError:
+        except subp.ProcessExecutionError:
             util.logexc(LOG, "Running interface command %s failed", cmd)
             return False
 
@@ -158,7 +159,7 @@ class Distro(distros.Distro):
         cmd.extend(pkglist)
 
         # Allow the output of this to flow outwards (ie not be captured)
-        util.subp(cmd, capture=False)
+        subp.subp(cmd, capture=False)
 
     def update_package_sources(self):
         self._runner.run("update-sources", self.package_command,
@@ -173,8 +174,8 @@ def _render_network(entries, target="/", conf_dir="etc/netctl",
 
     devs = []
     nameservers = []
-    resolv_conf = util.target_path(target, resolv_conf)
-    conf_dir = util.target_path(target, conf_dir)
+    resolv_conf = subp.target_path(target, resolv_conf)
+    conf_dir = subp.target_path(target, conf_dir)
 
     for (dev, info) in entries.items():
         if dev == 'lo':
diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py
index 37cf93bf..c2d1f77d 100644
--- a/cloudinit/distros/bsd.py
+++ b/cloudinit/distros/bsd.py
@@ -5,6 +5,7 @@ from cloudinit.distros import bsd_utils
 from cloudinit import helpers
 from cloudinit import log as logging
 from cloudinit import net
+from cloudinit import subp
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -50,7 +51,7 @@ class BSD(distros.Distro):
         else:
             group_add_cmd = self.group_add_cmd_prefix + [name]
             try:
-                util.subp(group_add_cmd)
+                subp.subp(group_add_cmd)
                 LOG.info("Created new group %s", name)
             except Exception:
                 util.logexc(LOG, "Failed to create group %s", name)
@@ -63,7 +64,7 @@ class BSD(distros.Distro):
                             "; user does not exist.", member, name)
                 continue
             try:
-                util.subp(self._get_add_member_to_group_cmd(member, name))
+                subp.subp(self._get_add_member_to_group_cmd(member, name))
                 LOG.info("Added user '%s' to group '%s'", member, name)
             except Exception:
                 util.logexc(LOG, "Failed to add user '%s' to group '%s'",
@@ -111,7 +112,7 @@ class BSD(distros.Distro):
         cmd.extend(pkglist)
 
         # Allow the output of this to flow outwards (ie not be captured)
-        util.subp(cmd, env=self._get_pkg_cmd_environ(), capture=False)
+        subp.subp(cmd, env=self._get_pkg_cmd_environ(), capture=False)
 
     def _write_network_config(self, netconfig):
         return self._supported_write_network_config(netconfig)
diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py
index 128bb523..844aaf21 100644
--- a/cloudinit/distros/debian.py
+++ b/cloudinit/distros/debian.py
@@ -13,6 +13,7 @@ import os
 from cloudinit import distros
 from cloudinit import helpers
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.distros.parsers.hostname import HostnameConf
@@ -197,7 +198,7 @@ class Distro(distros.Distro):
         # Allow the output of this to flow outwards (ie not be captured)
         util.log_time(logfunc=LOG.debug,
                       msg="apt-%s [%s]" % (command, ' '.join(cmd)),
-                      func=util.subp,
+                      func=subp.subp,
                       args=(cmd,), kwargs={'env': e, 'capture': False})
 
     def update_package_sources(self):
@@ -214,7 +215,7 @@ def _get_wrapper_prefix(cmd, mode):
 
     if (util.is_true(mode) or
         (str(mode).lower() == "auto" and cmd[0] and
-         util.which(cmd[0]))):
+         subp.which(cmd[0]))):
         return cmd
     else:
         return []
@@ -269,7 +270,7 @@ def update_locale_conf(locale, sys_path, keyname='LANG'):
     """Update system locale config"""
     LOG.debug('Updating %s with locale setting %s=%s',
               sys_path, keyname, locale)
-    util.subp(
+    subp.subp(
         ['update-locale', '--locale-file=' + sys_path,
          '%s=%s' % (keyname, locale)], capture=False)
 
@@ -291,7 +292,7 @@ def regenerate_locale(locale, sys_path, keyname='LANG'):
 
     # finally, trigger regeneration
     LOG.debug('Generating locales for %s', locale)
-    util.subp(['locale-gen', locale], capture=False)
+    subp.subp(['locale-gen', locale], capture=False)
 
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py
index b3a4ad67..dde34d41 100644
--- a/cloudinit/distros/freebsd.py
+++ b/cloudinit/distros/freebsd.py
@@ -10,6 +10,7 @@ from io import StringIO
 
 import cloudinit.distros.bsd
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 from cloudinit.settings import PER_INSTANCE
 
@@ -78,7 +79,7 @@ class Distro(cloudinit.distros.bsd.BSD):
         # Run the command
         LOG.info("Adding user %s", name)
         try:
-            util.subp(pw_useradd_cmd, logstring=log_pw_useradd_cmd)
+            subp.subp(pw_useradd_cmd, logstring=log_pw_useradd_cmd)
         except Exception:
             util.logexc(LOG, "Failed to create user %s", name)
             raise
@@ -90,7 +91,7 @@ class Distro(cloudinit.distros.bsd.BSD):
 
     def expire_passwd(self, user):
         try:
-            util.subp(['pw', 'usermod', user, '-p', '01-Jan-1970'])
+            subp.subp(['pw', 'usermod', user, '-p', '01-Jan-1970'])
         except Exception:
             util.logexc(LOG, "Failed to set pw expiration for %s", user)
             raise
@@ -102,7 +103,7 @@ class Distro(cloudinit.distros.bsd.BSD):
             hash_opt = "-h"
 
         try:
-            util.subp(['pw', 'usermod', user, hash_opt, '0'],
+            subp.subp(['pw', 'usermod', user, hash_opt, '0'],
                       data=passwd, logstring="chpasswd for %s" % user)
         except Exception:
             util.logexc(LOG, "Failed to set password for %s", user)
@@ -110,7 +111,7 @@ class Distro(cloudinit.distros.bsd.BSD):
 
     def lock_passwd(self, name):
         try:
-            util.subp(['pw', 'usermod', name, '-h', '-'])
+            subp.subp(['pw', 'usermod', name, '-h', '-'])
         except Exception:
             util.logexc(LOG, "Failed to lock user %s", name)
             raise
@@ -131,8 +132,8 @@ class Distro(cloudinit.distros.bsd.BSD):
 
         try:
             LOG.debug("Running cap_mkdb for %s", locale)
-            util.subp(['cap_mkdb', self.login_conf_fn])
-        except util.ProcessExecutionError:
+            subp.subp(['cap_mkdb', self.login_conf_fn])
+        except subp.ProcessExecutionError:
             # cap_mkdb failed, so restore the backup.
             util.logexc(LOG, "Failed to apply locale %s", locale)
             try:
diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py
index dc57717d..2bee1c89 100644
--- a/cloudinit/distros/gentoo.py
+++ b/cloudinit/distros/gentoo.py
@@ -9,6 +9,7 @@
 from cloudinit import distros
 from cloudinit import helpers
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.distros import net_util
@@ -39,7 +40,7 @@ class Distro(distros.Distro):
     def apply_locale(self, locale, out_fn=None):
         if not out_fn:
             out_fn = self.locale_conf_fn
-        util.subp(['locale-gen', '-G', locale], capture=False)
+        subp.subp(['locale-gen', '-G', locale], capture=False)
         # "" provides trailing newline during join
         lines = [
             util.make_header(),
@@ -94,11 +95,11 @@ class Distro(distros.Distro):
                 cmd = ['rc-update', 'add', 'net.{name}'.format(name=dev),
                        'default']
                 try:
-                    (_out, err) = util.subp(cmd)
+                    (_out, err) = subp.subp(cmd)
                     if len(err):
                         LOG.warning("Running %s resulted in stderr output: %s",
                                     cmd, err)
-                except util.ProcessExecutionError:
+                except subp.ProcessExecutionError:
                     util.logexc(LOG, "Running interface command %s failed",
                                 cmd)
 
@@ -119,12 +120,12 @@ class Distro(distros.Distro):
         LOG.debug("Attempting to run bring up interface %s using command %s",
                   device_name, cmd)
         try:
-            (_out, err) = util.subp(cmd)
+            (_out, err) = subp.subp(cmd)
             if len(err):
                 LOG.warning("Running %s resulted in stderr output: %s",
                             cmd, err)
             return True
-        except util.ProcessExecutionError:
+        except subp.ProcessExecutionError:
             util.logexc(LOG, "Running interface command %s failed", cmd)
             return False
 
@@ -137,11 +138,11 @@ class Distro(distros.Distro):
             # Grab device names from init scripts
             cmd = ['ls', '/etc/init.d/net.*']
             try:
-                (_out, err) = util.subp(cmd)
+                (_out, err) = subp.subp(cmd)
                 if len(err):
                     LOG.warning("Running %s resulted in stderr output: %s",
                                 cmd, err)
-            except util.ProcessExecutionError:
+            except subp.ProcessExecutionError:
                 util.logexc(LOG, "Running interface command %s failed", cmd)
                 return False
             devices = [x.split('.')[2] for x in _out.split('  ')]
@@ -208,7 +209,7 @@ class Distro(distros.Distro):
         cmd.extend(pkglist)
 
         # Allow the output of this to flow outwards (ie not be captured)
-        util.subp(cmd, capture=False)
+        subp.subp(cmd, capture=False)
 
     def update_package_sources(self):
         self._runner.run("update-sources", self.package_command,
diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py
index ecc8239a..066737a8 100644
--- a/cloudinit/distros/netbsd.py
+++ b/cloudinit/distros/netbsd.py
@@ -8,6 +8,7 @@ import platform
 
 import cloudinit.distros.bsd
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -76,7 +77,7 @@ class NetBSD(cloudinit.distros.bsd.BSD):
         # Run the command
         LOG.info("Adding user %s", name)
         try:
-            util.subp(adduser_cmd, logstring=log_adduser_cmd)
+            subp.subp(adduser_cmd, logstring=log_adduser_cmd)
         except Exception:
             util.logexc(LOG, "Failed to create user %s", name)
             raise
@@ -103,7 +104,7 @@ class NetBSD(cloudinit.distros.bsd.BSD):
                     crypt.mksalt(method))
 
         try:
-            util.subp(['usermod', '-p', hashed_pw, user])
+            subp.subp(['usermod', '-p', hashed_pw, user])
         except Exception:
             util.logexc(LOG, "Failed to set password for %s", user)
             raise
@@ -111,21 +112,21 @@ class NetBSD(cloudinit.distros.bsd.BSD):
 
     def force_passwd_change(self, user):
         try:
-            util.subp(['usermod', '-F', user])
+            subp.subp(['usermod', '-F', user])
         except Exception:
             util.logexc(LOG, "Failed to set pw expiration for %s", user)
             raise
 
     def lock_passwd(self, name):
         try:
-            util.subp(['usermod', '-C', 'yes', name])
+            subp.subp(['usermod', '-C', 'yes', name])
         except Exception:
             util.logexc(LOG, "Failed to lock user %s", name)
             raise
 
     def unlock_passwd(self, name):
         try:
-            util.subp(['usermod', '-C', 'no', name])
+            subp.subp(['usermod', '-C', 'no', name])
         except Exception:
             util.logexc(LOG, "Failed to unlock user %s", name)
             raise
diff --git a/cloudinit/distros/openbsd.py b/cloudinit/distros/openbsd.py
index ca094156..07c76530 100644
--- a/cloudinit/distros/openbsd.py
+++ b/cloudinit/distros/openbsd.py
@@ -7,6 +7,7 @@ import platform
 
 import cloudinit.distros.netbsd
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -27,7 +28,7 @@ class Distro(cloudinit.distros.netbsd.NetBSD):
 
     def lock_passwd(self, name):
         try:
-            util.subp(['usermod', '-p', '*', name])
+            subp.subp(['usermod', '-p', '*', name])
         except Exception:
             util.logexc(LOG, "Failed to lock user %s", name)
             raise
diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py
index 68028d20..ffb7d0e8 100644
--- a/cloudinit/distros/opensuse.py
+++ b/cloudinit/distros/opensuse.py
@@ -14,6 +14,7 @@ from cloudinit.distros.parsers.hostname import HostnameConf
 
 from cloudinit import helpers
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.distros import rhel_util as rhutil
@@ -97,7 +98,7 @@ class Distro(distros.Distro):
         cmd.extend(pkglist)
 
         # Allow the output of this to flow outwards (ie not be captured)
-        util.subp(cmd, capture=False)
+        subp.subp(cmd, capture=False)
 
     def set_timezone(self, tz):
         tz_file = self._find_tz_file(tz)
@@ -129,7 +130,7 @@ class Distro(distros.Distro):
         if self.uses_systemd() and filename.endswith('/previous-hostname'):
             return util.load_file(filename).strip()
         elif self.uses_systemd():
-            (out, _err) = util.subp(['hostname'])
+            (out, _err) = subp.subp(['hostname'])
             if len(out):
                 return out
             else:
@@ -163,7 +164,7 @@ class Distro(distros.Distro):
         if self.uses_systemd() and out_fn.endswith('/previous-hostname'):
             util.write_file(out_fn, hostname)
         elif self.uses_systemd():
-            util.subp(['hostnamectl', 'set-hostname', str(hostname)])
+            subp.subp(['hostnamectl', 'set-hostname', str(hostname)])
         else:
             conf = None
             try:
diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py
index f55d96f7..c72f7c17 100644
--- a/cloudinit/distros/rhel.py
+++ b/cloudinit/distros/rhel.py
@@ -11,6 +11,7 @@
 from cloudinit import distros
 from cloudinit import helpers
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.distros import rhel_util
@@ -83,7 +84,7 @@ class Distro(distros.Distro):
         if self.uses_systemd() and out_fn.endswith('/previous-hostname'):
             util.write_file(out_fn, hostname)
         elif self.uses_systemd():
-            util.subp(['hostnamectl', 'set-hostname', str(hostname)])
+            subp.subp(['hostnamectl', 'set-hostname', str(hostname)])
         else:
             host_cfg = {
                 'HOSTNAME': hostname,
@@ -108,7 +109,7 @@ class Distro(distros.Distro):
         if self.uses_systemd() and filename.endswith('/previous-hostname'):
             return util.load_file(filename).strip()
         elif self.uses_systemd():
-            (out, _err) = util.subp(['hostname'])
+            (out, _err) = subp.subp(['hostname'])
             if len(out):
                 return out
             else:
@@ -146,7 +147,7 @@ class Distro(distros.Distro):
         if pkgs is None:
             pkgs = []
 
-        if util.which('dnf'):
+        if subp.which('dnf'):
             LOG.debug('Using DNF for package management')
             cmd = ['dnf']
         else:
@@ -173,7 +174,7 @@ class Distro(distros.Distro):
         cmd.extend(pkglist)
 
         # Allow the output of this to flow outwards (ie not be captured)
-        util.subp(cmd, capture=False)
+        subp.subp(cmd, capture=False)
 
     def update_package_sources(self):
         self._runner.run("update-sources", self.package_command,
diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py
index 7fe17a2e..72b5ac59 100644
--- a/cloudinit/gpg.py
+++ b/cloudinit/gpg.py
@@ -8,7 +8,7 @@
 """gpg.py - Collection of gpg key related functions"""
 
 from cloudinit import log as logging
-from cloudinit import util
+from cloudinit import subp
 
 import time
 
@@ -18,9 +18,9 @@ LOG = logging.getLogger(__name__)
 def export_armour(key):
     """Export gpg key, armoured key gets returned"""
     try:
-        (armour, _) = util.subp(["gpg", "--export", "--armour", key],
+        (armour, _) = subp.subp(["gpg", "--export", "--armour", key],
                                 capture=True)
-    except util.ProcessExecutionError as error:
+    except subp.ProcessExecutionError as error:
         # debug, since it happens for any key not on the system initially
         LOG.debug('Failed to export armoured key "%s": %s', key, error)
         armour = None
@@ -51,11 +51,11 @@ def recv_key(key, keyserver, retries=(1, 1)):
     while True:
         trynum += 1
         try:
-            util.subp(cmd, capture=True)
+            subp.subp(cmd, capture=True)
             LOG.debug("Imported key '%s' from keyserver '%s' on try %d",
                       key, keyserver, trynum)
             return
-        except util.ProcessExecutionError as e:
+        except subp.ProcessExecutionError as e:
             error = e
         try:
             naplen = next(sleeps)
@@ -72,9 +72,9 @@ def recv_key(key, keyserver, retries=(1, 1)):
 def delete_key(key):
     """Delete the specified key from the local gpg ring"""
     try:
-        util.subp(["gpg", "--batch", "--yes", "--delete-keys", key],
+        subp.subp(["gpg", "--batch", "--yes", "--delete-keys", key],
                   capture=True)
-    except util.ProcessExecutionError as error:
+    except subp.ProcessExecutionError as error:
         LOG.warning('Failed delete key "%s": %s', key, error)
 
 
diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py
index dca50a49..c6205097 100644
--- a/cloudinit/handlers/boot_hook.py
+++ b/cloudinit/handlers/boot_hook.py
@@ -12,6 +12,7 @@ import os
 
 from cloudinit import handlers
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.settings import (PER_ALWAYS)
@@ -48,8 +49,8 @@ class BootHookPartHandler(handlers.Handler):
             env = os.environ.copy()
             if self.instance_id is not None:
                 env['INSTANCE_ID'] = str(self.instance_id)
-            util.subp([filepath], env=env)
-        except util.ProcessExecutionError:
+            subp.subp([filepath], env=env)
+        except subp.ProcessExecutionError:
             util.logexc(LOG, "Boothooks script %s execution error", filepath)
         except Exception:
             util.logexc(LOG, "Boothooks unknown error when running %s",
diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py
index 003cad60..a9d29537 100644
--- a/cloudinit/handlers/upstart_job.py
+++ b/cloudinit/handlers/upstart_job.py
@@ -13,6 +13,7 @@ import re
 
 from cloudinit import handlers
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.settings import (PER_INSTANCE)
@@ -52,7 +53,7 @@ class UpstartJobPartHandler(handlers.Handler):
         util.write_file(path, payload, 0o644)
 
         if SUITABLE_UPSTART:
-            util.subp(["initctl", "reload-configuration"], capture=False)
+            subp.subp(["initctl", "reload-configuration"], capture=False)
 
 
 def _has_suitable_upstart():
@@ -63,7 +64,7 @@ def _has_suitable_upstart():
     if not os.path.exists("/sbin/initctl"):
         return False
     try:
-        (version_out, _err) = util.subp(["initctl", "version"])
+        (version_out, _err) = subp.subp(["initctl", "version"])
     except Exception:
         util.logexc(LOG, "initctl version failed")
         return False
@@ -77,7 +78,7 @@ def _has_suitable_upstart():
         if not os.path.exists("/usr/bin/dpkg-query"):
             return False
         try:
-            (dpkg_ver, _err) = util.subp(["dpkg-query",
+            (dpkg_ver, _err) = subp.subp(["dpkg-query",
                                           "--showformat=${Version}",
                                           "--show", "upstart"], rcs=[0, 1])
         except Exception:
@@ -86,9 +87,9 @@ def _has_suitable_upstart():
 
         try:
             good = "1.8-0ubuntu1.2"
-            util.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good])
+            subp.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good])
             return True
-        except util.ProcessExecutionError as e:
+        except subp.ProcessExecutionError as e:
             if e.exit_code == 1:
                 pass
             else:
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 8af24fa9..a57fea0a 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -12,6 +12,7 @@ import os
 import re
 from functools import partial
 
+from cloudinit import subp
 from cloudinit import util
 from cloudinit.net.network_state import mask_to_net_prefix
 from cloudinit.url_helper import UrlError, readurl
@@ -358,7 +359,7 @@ def find_fallback_nic_on_freebsd(blacklist_drivers=None):
 
     we'll use the first interface from ``ifconfig -l -u ether``
     """
-    stdout, _stderr = util.subp(['ifconfig', '-l', '-u', 'ether'])
+    stdout, _stderr = subp.subp(['ifconfig', '-l', '-u', 'ether'])
     values = stdout.split()
     if values:
         return values[0]
@@ -620,9 +621,9 @@ def _get_current_rename_info(check_downable=True):
 
     if check_downable:
         nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]")
-        ipv6, _err = util.subp(['ip', '-6', 'addr', 'show', 'permanent',
+        ipv6, _err = subp.subp(['ip', '-6', 'addr', 'show', 'permanent',
                                 'scope', 'global'], capture=True)
-        ipv4, _err = util.subp(['ip', '-4', 'addr', 'show'], capture=True)
+        ipv4, _err = subp.subp(['ip', '-4', 'addr', 'show'], capture=True)
 
         nics_with_addresses = set()
         for bytes_out in (ipv6, ipv4):
@@ -658,13 +659,13 @@ def _rename_interfaces(renames, strict_present=True, strict_busy=True,
                     for data in cur_info.values())
 
     def rename(cur, new):
-        util.subp(["ip", "link", "set", cur, "name", new], capture=True)
+        subp.subp(["ip", "link", "set", cur, "name", new], capture=True)
 
     def down(name):
-        util.subp(["ip", "link", "set", name, "down"], capture=True)
+        subp.subp(["ip", "link", "set", name, "down"], capture=True)
 
     def up(name):
-        util.subp(["ip", "link", "set", name, "up"], capture=True)
+        subp.subp(["ip", "link", "set", name, "up"], capture=True)
 
     ops = []
     errors = []
@@ -819,7 +820,7 @@ def get_interfaces_by_mac():
 
 
 def get_interfaces_by_mac_on_freebsd():
-    (out, _) = util.subp(['ifconfig', '-a', 'ether'])
+    (out, _) = subp.subp(['ifconfig', '-a', 'ether'])
 
     # flatten each interface block in a single line
     def flatten(out):
@@ -850,7 +851,7 @@ def get_interfaces_by_mac_on_netbsd():
     re_field_match = (
             r"(?P<ifname>\w+).*address:\s"
             r"(?P<mac>([\da-f]{2}[:-]){5}([\da-f]{2})).*")
-    (out, _) = util.subp(['ifconfig', '-a'])
+    (out, _) = subp.subp(['ifconfig', '-a'])
     if_lines = re.sub(r'\n\s+', ' ', out).splitlines()
     for line in if_lines:
         m = re.match(re_field_match, line)
@@ -865,7 +866,7 @@ def get_interfaces_by_mac_on_openbsd():
     re_field_match = (
         r"(?P<ifname>\w+).*lladdr\s"
         r"(?P<mac>([\da-f]{2}[:-]){5}([\da-f]{2})).*")
-    (out, _) = util.subp(['ifconfig', '-a'])
+    (out, _) = subp.subp(['ifconfig', '-a'])
     if_lines = re.sub(r'\n\s+', ' ', out).splitlines()
     for line in if_lines:
         m = re.match(re_field_match, line)
@@ -1067,11 +1068,11 @@ class EphemeralIPv4Network(object):
     def __exit__(self, excp_type, excp_value, excp_traceback):
         """Teardown anything we set up."""
         for cmd in self.cleanup_cmds:
-            util.subp(cmd, capture=True)
+            subp.subp(cmd, capture=True)
 
     def _delete_address(self, address, prefix):
         """Perform the ip command to remove the specified address."""
-        util.subp(
+        subp.subp(
             ['ip', '-family', 'inet', 'addr', 'del',
              '%s/%s' % (address, prefix), 'dev', self.interface],
             capture=True)
@@ -1083,11 +1084,11 @@ class EphemeralIPv4Network(object):
             'Attempting setup of ephemeral network on %s with %s brd %s',
             self.interface, cidr, self.broadcast)
         try:
-            util.subp(
+            subp.subp(
                 ['ip', '-family', 'inet', 'addr', 'add', cidr, 'broadcast',
                  self.broadcast, 'dev', self.interface],
                 capture=True, update_env={'LANG': 'C'})
-        except util.ProcessExecutionError as e:
+        except subp.ProcessExecutionError as e:
             if "File exists" not in e.stderr:
                 raise
             LOG.debug(
@@ -1095,7 +1096,7 @@ class EphemeralIPv4Network(object):
                 self.interface, self.ip)
         else:
             # Address creation success, bring up device and queue cleanup
-            util.subp(
+            subp.subp(
                 ['ip', '-family', 'inet', 'link', 'set', 'dev', self.interface,
                  'up'], capture=True)
             self.cleanup_cmds.append(
@@ -1112,7 +1113,7 @@ class EphemeralIPv4Network(object):
             via_arg = []
             if gateway != "0.0.0.0/0":
                 via_arg = ['via', gateway]
-            util.subp(
+            subp.subp(
                 ['ip', '-4', 'route', 'add', net_address] + via_arg +
                 ['dev', self.interface], capture=True)
             self.cleanup_cmds.insert(
@@ -1122,20 +1123,20 @@ class EphemeralIPv4Network(object):
     def _bringup_router(self):
         """Perform the ip commands to fully setup the router if needed."""
         # Check if a default route exists and exit if it does
-        out, _ = util.subp(['ip', 'route', 'show', '0.0.0.0/0'], capture=True)
+        out, _ = subp.subp(['ip', 'route', 'show', '0.0.0.0/0'], capture=True)
         if 'default' in out:
             LOG.debug(
                 'Skip ephemeral route setup. %s already has default route: %s',
                 self.interface, out.strip())
             return
-        util.subp(
+        subp.subp(
             ['ip', '-4', 'route', 'add', self.router, 'dev', self.interface,
              'src', self.ip], capture=True)
         self.cleanup_cmds.insert(
             0,
             ['ip', '-4', 'route', 'del', self.router, 'dev', self.interface,
              'src', self.ip])
-        util.subp(
+        subp.subp(
             ['ip', '-4', 'route', 'add', 'default', 'via', self.router,
              'dev', self.interface], capture=True)
         self.cleanup_cmds.insert(
diff --git a/cloudinit/net/bsd.py b/cloudinit/net/bsd.py
index fb714d4c..1c355a98 100644
--- a/cloudinit/net/bsd.py
+++ b/cloudinit/net/bsd.py
@@ -5,6 +5,7 @@ import re
 from cloudinit import log as logging
 from cloudinit import net
 from cloudinit import util
+from cloudinit import subp
 from cloudinit.distros.parsers.resolv_conf import ResolvConf
 from cloudinit.distros import bsd_utils
 
@@ -18,11 +19,11 @@ class BSDRenderer(renderer.Renderer):
     rc_conf_fn = 'etc/rc.conf'
 
     def get_rc_config_value(self, key):
-        fn = util.target_path(self.target, self.rc_conf_fn)
+        fn = subp.target_path(self.target, self.rc_conf_fn)
         bsd_utils.get_rc_config_value(key, fn=fn)
 
     def set_rc_config_value(self, key, value):
-        fn = util.target_path(self.target, self.rc_conf_fn)
+        fn = subp.target_path(self.target, self.rc_conf_fn)
         bsd_utils.set_rc_config_value(key, value, fn=fn)
 
     def __init__(self, config=None):
@@ -111,12 +112,12 @@ class BSDRenderer(renderer.Renderer):
         # Try to read the /etc/resolv.conf or just start from scratch if that
         # fails.
         try:
-            resolvconf = ResolvConf(util.load_file(util.target_path(
+            resolvconf = ResolvConf(util.load_file(subp.target_path(
                 target, self.resolv_conf_fn)))
             resolvconf.parse()
         except IOError:
             util.logexc(LOG, "Failed to parse %s, use new empty file",
-                        util.target_path(target, self.resolv_conf_fn))
+                        subp.target_path(target, self.resolv_conf_fn))
             resolvconf = ResolvConf('')
             resolvconf.parse()
 
@@ -134,7 +135,7 @@ class BSDRenderer(renderer.Renderer):
             except ValueError:
                 util.logexc(LOG, "Failed to add search domain %s", domain)
         util.write_file(
-            util.target_path(target, self.resolv_conf_fn),
+            subp.target_path(target, self.resolv_conf_fn),
             str(resolvconf), 0o644)
 
     def render_network_state(self, network_state, templates=None, target=None):
diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py
index 19d0199c..d03baeab 100644
--- a/cloudinit/net/dhcp.py
+++ b/cloudinit/net/dhcp.py
@@ -17,6 +17,7 @@ from cloudinit.net import (
     has_url_connectivity)
 from cloudinit.net.network_state import mask_and_ipv4_to_bcast_addr as bcip
 from cloudinit import temp_utils
+from cloudinit import subp
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -150,7 +151,7 @@ def maybe_perform_dhcp_discovery(nic=None):
         LOG.debug(
             'Skip dhcp_discovery: nic %s not found in get_devicelist.', nic)
         return []
-    dhclient_path = util.which('dhclient')
+    dhclient_path = subp.which('dhclient')
     if not dhclient_path:
         LOG.debug('Skip dhclient configuration: No dhclient command found.')
         return []
@@ -219,10 +220,10 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir):
     # Generally dhclient relies on dhclient-script PREINIT action to bring the
     # link up before attempting discovery. Since we are using -sf /bin/true,
     # we need to do that "link up" ourselves first.
-    util.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True)
+    subp.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True)
     cmd = [sandbox_dhclient_cmd, '-1', '-v', '-lf', lease_file,
            '-pf', pid_file, interface, '-sf', '/bin/true']
-    util.subp(cmd, capture=True)
+    subp.subp(cmd, capture=True)
 
     # Wait for pid file and lease file to appear, and for the process
     # named by the pid file to daemonize (have pid 1 as its parent). If we
diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py
index 2f714563..b4c69457 100644
--- a/cloudinit/net/eni.py
+++ b/cloudinit/net/eni.py
@@ -11,6 +11,7 @@ from . import renderer
 from .network_state import subnet_is_ipv6
 
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 
 
@@ -511,13 +512,13 @@ class Renderer(renderer.Renderer):
         return '\n\n'.join(['\n'.join(s) for s in sections]) + "\n"
 
     def render_network_state(self, network_state, templates=None, target=None):
-        fpeni = util.target_path(target, self.eni_path)
+        fpeni = subp.target_path(target, self.eni_path)
         util.ensure_dir(os.path.dirname(fpeni))
         header = self.eni_header if self.eni_header else ""
         util.write_file(fpeni, header + self._render_interfaces(network_state))
 
         if self.netrules_path:
-            netrules = util.target_path(target, self.netrules_path)
+            netrules = subp.target_path(target, self.netrules_path)
             util.ensure_dir(os.path.dirname(netrules))
             util.write_file(netrules,
                             self._render_persistent_net(network_state))
@@ -544,9 +545,9 @@ def available(target=None):
     expected = ['ifquery', 'ifup', 'ifdown']
     search = ['/sbin', '/usr/sbin']
     for p in expected:
-        if not util.which(p, search=search, target=target):
+        if not subp.which(p, search=search, target=target):
             return False
-    eni = util.target_path(target, 'etc/network/interfaces')
+    eni = subp.target_path(target, 'etc/network/interfaces')
     if not os.path.isfile(eni):
         return False
 
diff --git a/cloudinit/net/freebsd.py b/cloudinit/net/freebsd.py
index 60f05bb2..0285dfec 100644
--- a/cloudinit/net/freebsd.py
+++ b/cloudinit/net/freebsd.py
@@ -2,6 +2,7 @@
 
 from cloudinit import log as logging
 import cloudinit.net.bsd
+from cloudinit import subp
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -30,17 +31,17 @@ class Renderer(cloudinit.net.bsd.BSDRenderer):
             LOG.debug("freebsd generate postcmd disabled")
             return
 
-        util.subp(['service', 'netif', 'restart'], capture=True)
+        subp.subp(['service', 'netif', 'restart'], capture=True)
         # On FreeBSD 10, the restart of routing and dhclient is likely to fail
         # because
         # - routing: it cannot remove the loopback route, but it will still set
         #   up the default route as expected.
         # - dhclient: it cannot stop the dhclient started by the netif service.
         # In both case, the situation is ok, and we can proceed.
-        util.subp(['service', 'routing', 'restart'], capture=True, rcs=[0, 1])
+        subp.subp(['service', 'routing', 'restart'], capture=True, rcs=[0, 1])
 
         for dhcp_interface in self.dhcp_interfaces():
-            util.subp(['service', 'dhclient', 'restart', dhcp_interface],
+            subp.subp(['service', 'dhclient', 'restart', dhcp_interface],
                       rcs=[0, 1],
                       capture=True)
 
diff --git a/cloudinit/net/netbsd.py b/cloudinit/net/netbsd.py
index 9cc8ef31..30437b5f 100644
--- a/cloudinit/net/netbsd.py
+++ b/cloudinit/net/netbsd.py
@@ -1,6 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 import cloudinit.net.bsd
 
@@ -29,9 +30,9 @@ class Renderer(cloudinit.net.bsd.BSDRenderer):
             LOG.debug("netbsd generate postcmd disabled")
             return
 
-        util.subp(['service', 'network', 'restart'], capture=True)
+        subp.subp(['service', 'network', 'restart'], capture=True)
         if self.dhcp_interfaces():
-            util.subp(['service', 'dhcpcd', 'restart'], capture=True)
+            subp.subp(['service', 'dhcpcd', 'restart'], capture=True)
 
     def set_route(self, network, netmask, gateway):
         if network == '0.0.0.0':
diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py
index 89855270..53347c83 100644
--- a/cloudinit/net/netplan.py
+++ b/cloudinit/net/netplan.py
@@ -8,6 +8,7 @@ from .network_state import subnet_is_ipv6, NET_CONFIG_TO_V2, IPV6_DYNAMIC_TYPES
 
 from cloudinit import log as logging
 from cloudinit import util
+from cloudinit import subp
 from cloudinit import safeyaml
 from cloudinit.net import SYS_CLASS_NET, get_devicelist
 
@@ -164,14 +165,14 @@ def _extract_bond_slaves_by_name(interfaces, entry, bond_master):
 def _clean_default(target=None):
     # clean out any known default files and derived files in target
     # LP: #1675576
-    tpath = util.target_path(target, "etc/netplan/00-snapd-config.yaml")
+    tpath = subp.target_path(target, "etc/netplan/00-snapd-config.yaml")
     if not os.path.isfile(tpath):
         return
     content = util.load_file(tpath, decode=False)
     if content != KNOWN_SNAPD_CONFIG:
         return
 
-    derived = [util.target_path(target, f) for f in (
+    derived = [subp.target_path(target, f) for f in (
                'run/systemd/network/10-netplan-all-en.network',
                'run/systemd/network/10-netplan-all-eth.network',
                'run/systemd/generator/netplan.stamp')]
@@ -203,10 +204,10 @@ class Renderer(renderer.Renderer):
     def features(self):
         if self._features is None:
             try:
-                info_blob, _err = util.subp(self.NETPLAN_INFO, capture=True)
+                info_blob, _err = subp.subp(self.NETPLAN_INFO, capture=True)
                 info = util.load_yaml(info_blob)
                 self._features = info['netplan.io']['features']
-            except util.ProcessExecutionError:
+            except subp.ProcessExecutionError:
                 # if the info subcommand is not present then we don't have any
                 # new features
                 pass
@@ -218,7 +219,7 @@ class Renderer(renderer.Renderer):
         # check network state for version
         # if v2, then extract network_state.config
         # else render_v2_from_state
-        fpnplan = os.path.join(util.target_path(target), self.netplan_path)
+        fpnplan = os.path.join(subp.target_path(target), self.netplan_path)
 
         util.ensure_dir(os.path.dirname(fpnplan))
         header = self.netplan_header if self.netplan_header else ""
@@ -239,7 +240,7 @@ class Renderer(renderer.Renderer):
         if not run:
             LOG.debug("netplan generate postcmd disabled")
             return
-        util.subp(self.NETPLAN_GENERATE, capture=True)
+        subp.subp(self.NETPLAN_GENERATE, capture=True)
 
     def _net_setup_link(self, run=False):
         """To ensure device link properties are applied, we poke
@@ -253,7 +254,7 @@ class Renderer(renderer.Renderer):
         for cmd in [setup_lnk + [SYS_CLASS_NET + iface]
                     for iface in get_devicelist() if
                     os.path.islink(SYS_CLASS_NET + iface)]:
-            util.subp(cmd, capture=True)
+            subp.subp(cmd, capture=True)
 
     def _render_content(self, network_state):
 
@@ -406,7 +407,7 @@ def available(target=None):
     expected = ['netplan']
     search = ['/usr/sbin', '/sbin']
     for p in expected:
-        if not util.which(p, search=search, target=target):
+        if not subp.which(p, search=search, target=target):
             return False
     return True
 
diff --git a/cloudinit/net/openbsd.py b/cloudinit/net/openbsd.py
index b9897e90..489ea48b 100644
--- a/cloudinit/net/openbsd.py
+++ b/cloudinit/net/openbsd.py
@@ -1,6 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import util
 import cloudinit.net.bsd
 
@@ -12,7 +13,7 @@ class Renderer(cloudinit.net.bsd.BSDRenderer):
     def write_config(self):
         for device_name, v in self.interface_configurations.items():
             if_file = 'etc/hostname.{}'.format(device_name)
-            fn = util.target_path(self.target, if_file)
+            fn = subp.target_path(self.target, if_file)
             if device_name in self.dhcp_interfaces():
                 content = 'dhcp\n'
             elif isinstance(v, dict):
@@ -30,12 +31,12 @@ class Renderer(cloudinit.net.bsd.BSDRenderer):
         if not self._postcmds:
             LOG.debug("openbsd generate postcmd disabled")
             return
-        util.subp(['sh', '/etc/netstart'], capture=True)
+        subp.subp(['sh', '/etc/netstart'], capture=True)
 
     def set_route(self, network, netmask, gateway):
         if network == '0.0.0.0':
             if_file = 'etc/mygate'
-            fn = util.target_path(self.target, if_file)
+            fn = subp.target_path(self.target, if_file)
             content = gateway + '\n'
             util.write_file(fn, content)
 
diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py
index 0a387377..f36c300f 100644
--- a/cloudinit/net/sysconfig.py
+++ b/cloudinit/net/sysconfig.py
@@ -9,6 +9,7 @@ from configobj import ConfigObj
 
 from cloudinit import log as logging
 from cloudinit import util
+from cloudinit import subp
 from cloudinit.distros.parsers import networkmanager_conf
 from cloudinit.distros.parsers import resolv_conf
 
@@ -858,19 +859,19 @@ class Renderer(renderer.Renderer):
         if not templates:
             templates = self.templates
         file_mode = 0o644
-        base_sysconf_dir = util.target_path(target, self.sysconf_dir)
+        base_sysconf_dir = subp.target_path(target, self.sysconf_dir)
         for path, data in self._render_sysconfig(base_sysconf_dir,
                                                  network_state, self.flavor,
                                                  templates=templates).items():
             util.write_file(path, data, file_mode)
         if self.dns_path:
-            dns_path = util.target_path(target, self.dns_path)
+            dns_path = subp.target_path(target, self.dns_path)
             resolv_content = self._render_dns(network_state,
                                               existing_dns_path=dns_path)
             if resolv_content:
                 util.write_file(dns_path, resolv_content, file_mode)
         if self.networkmanager_conf_path:
-            nm_conf_path = util.target_path(target,
+            nm_conf_path = subp.target_path(target,
                                             self.networkmanager_conf_path)
             nm_conf_content = self._render_networkmanager_conf(network_state,
                                                                templates)
@@ -878,12 +879,12 @@ class Renderer(renderer.Renderer):
                 util.write_file(nm_conf_path, nm_conf_content, file_mode)
         if self.netrules_path:
             netrules_content = self._render_persistent_net(network_state)
-            netrules_path = util.target_path(target, self.netrules_path)
+            netrules_path = subp.target_path(target, self.netrules_path)
             util.write_file(netrules_path, netrules_content, file_mode)
         if available_nm(target=target):
-            enable_ifcfg_rh(util.target_path(target, path=NM_CFG_FILE))
+            enable_ifcfg_rh(subp.target_path(target, path=NM_CFG_FILE))
 
-        sysconfig_path = util.target_path(target, templates.get('control'))
+        sysconfig_path = subp.target_path(target, templates.get('control'))
         # Distros configuring /etc/sysconfig/network as a file e.g. Centos
         if sysconfig_path.endswith('network'):
             util.ensure_dir(os.path.dirname(sysconfig_path))
@@ -906,20 +907,20 @@ def available_sysconfig(target=None):
     expected = ['ifup', 'ifdown']
     search = ['/sbin', '/usr/sbin']
     for p in expected:
-        if not util.which(p, search=search, target=target):
+        if not subp.which(p, search=search, target=target):
             return False
 
     expected_paths = [
         'etc/sysconfig/network-scripts/network-functions',
         'etc/sysconfig/config']
     for p in expected_paths:
-        if os.path.isfile(util.target_path(target, p)):
+        if os.path.isfile(subp.target_path(target, p)):
             return True
     return False
 
 
 def available_nm(target=None):
-    if not os.path.isfile(util.target_path(target, path=NM_CFG_FILE)):
+    if not os.path.isfile(subp.target_path(target, path=NM_CFG_FILE)):
         return False
     return True
 
diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py
index 7768da7c..d4881592 100644
--- a/cloudinit/net/tests/test_dhcp.py
+++ b/cloudinit/net/tests/test_dhcp.py
@@ -266,7 +266,7 @@ class TestDHCPDiscoveryClean(CiTestCase):
             'Skip dhcp_discovery: nic idontexist not found in get_devicelist.',
             self.logs.getvalue())
 
-    @mock.patch('cloudinit.net.dhcp.util.which')
+    @mock.patch('cloudinit.net.dhcp.subp.which')
     @mock.patch('cloudinit.net.dhcp.find_fallback_nic')
     def test_absent_dhclient_command(self, m_fallback, m_which):
         """When dhclient doesn't exist in the OS, log the issue and no-op."""
@@ -279,7 +279,7 @@ class TestDHCPDiscoveryClean(CiTestCase):
 
     @mock.patch('cloudinit.temp_utils.os.getuid')
     @mock.patch('cloudinit.net.dhcp.dhcp_discovery')
-    @mock.patch('cloudinit.net.dhcp.util.which')
+    @mock.patch('cloudinit.net.dhcp.subp.which')
     @mock.patch('cloudinit.net.dhcp.find_fallback_nic')
     def test_dhclient_run_with_tmpdir(self, m_fback, m_which, m_dhcp, m_uid):
         """maybe_perform_dhcp_discovery passes tmpdir to dhcp_discovery."""
@@ -302,7 +302,7 @@ class TestDHCPDiscoveryClean(CiTestCase):
 
     @mock.patch('time.sleep', mock.MagicMock())
     @mock.patch('cloudinit.net.dhcp.os.kill')
-    @mock.patch('cloudinit.net.dhcp.util.subp')
+    @mock.patch('cloudinit.net.dhcp.subp.subp')
     def test_dhcp_discovery_run_in_sandbox_warns_invalid_pid(self, m_subp,
                                                              m_kill):
         """dhcp_discovery logs a warning when pidfile contains invalid content.
@@ -337,7 +337,7 @@ class TestDHCPDiscoveryClean(CiTestCase):
     @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid')
     @mock.patch('cloudinit.net.dhcp.os.kill')
     @mock.patch('cloudinit.net.dhcp.util.wait_for_files')
-    @mock.patch('cloudinit.net.dhcp.util.subp')
+    @mock.patch('cloudinit.net.dhcp.subp.subp')
     def test_dhcp_discovery_run_in_sandbox_waits_on_lease_and_pid(self,
                                                                   m_subp,
                                                                   m_wait,
@@ -364,7 +364,7 @@ class TestDHCPDiscoveryClean(CiTestCase):
 
     @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid')
     @mock.patch('cloudinit.net.dhcp.os.kill')
-    @mock.patch('cloudinit.net.dhcp.util.subp')
+    @mock.patch('cloudinit.net.dhcp.subp.subp')
     def test_dhcp_discovery_run_in_sandbox(self, m_subp, m_kill, m_getppid):
         """dhcp_discovery brings up the interface and runs dhclient.
 
@@ -529,7 +529,7 @@ class TestEphemeralDhcpNoNetworkSetup(HttprettyTestCase):
         # Ensure that no teardown happens:
         m_dhcp.assert_not_called()
 
-    @mock.patch('cloudinit.net.dhcp.util.subp')
+    @mock.patch('cloudinit.net.dhcp.subp.subp')
     @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
     def test_ephemeral_dhcp_setup_network_if_url_connectivity(
             self, m_dhcp, m_subp):
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index 835ed807..1c0a16a8 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -14,7 +14,8 @@ import requests
 import cloudinit.net as net
 from cloudinit import safeyaml as yaml
 from cloudinit.tests.helpers import CiTestCase, HttprettyTestCase
-from cloudinit.util import ProcessExecutionError, ensure_file, write_file
+from cloudinit.subp import ProcessExecutionError
+from cloudinit.util import ensure_file, write_file
 
 
 class TestSysDevPath(CiTestCase):
@@ -541,7 +542,7 @@ class TestInterfaceHasOwnMAC(CiTestCase):
             net.interface_has_own_mac('eth1', strict=True)
 
 
-@mock.patch('cloudinit.net.util.subp')
+@mock.patch('cloudinit.net.subp.subp')
 class TestEphemeralIPV4Network(CiTestCase):
 
     with_logs = True
diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py
index 1001f149..628e2908 100644
--- a/cloudinit/netinfo.py
+++ b/cloudinit/netinfo.py
@@ -13,6 +13,7 @@ import re
 
 from cloudinit import log as logging
 from cloudinit.net.network_state import net_prefix_to_ipv4_mask
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.simpletable import SimpleTable
@@ -197,15 +198,15 @@ def _netdev_info_ifconfig(ifconfig_data):
 def netdev_info(empty=""):
     devs = {}
     if util.is_NetBSD():
-        (ifcfg_out, _err) = util.subp(["ifconfig", "-a"], rcs=[0, 1])
+        (ifcfg_out, _err) = subp.subp(["ifconfig", "-a"], rcs=[0, 1])
         devs = _netdev_info_ifconfig_netbsd(ifcfg_out)
-    elif util.which('ip'):
+    elif subp.which('ip'):
         # Try iproute first of all
-        (ipaddr_out, _err) = util.subp(["ip", "addr", "show"])
+        (ipaddr_out, _err) = subp.subp(["ip", "addr", "show"])
         devs = _netdev_info_iproute(ipaddr_out)
-    elif util.which('ifconfig'):
+    elif subp.which('ifconfig'):
         # Fall back to net-tools if iproute2 is not present
-        (ifcfg_out, _err) = util.subp(["ifconfig", "-a"], rcs=[0, 1])
+        (ifcfg_out, _err) = subp.subp(["ifconfig", "-a"], rcs=[0, 1])
         devs = _netdev_info_ifconfig(ifcfg_out)
     else:
         LOG.warning(
@@ -285,10 +286,10 @@ def _netdev_route_info_iproute(iproute_data):
         entry['flags'] = ''.join(flags)
         routes['ipv4'].append(entry)
     try:
-        (iproute_data6, _err6) = util.subp(
+        (iproute_data6, _err6) = subp.subp(
             ["ip", "--oneline", "-6", "route", "list", "table", "all"],
             rcs=[0, 1])
-    except util.ProcessExecutionError:
+    except subp.ProcessExecutionError:
         pass
     else:
         entries6 = iproute_data6.splitlines()
@@ -357,9 +358,9 @@ def _netdev_route_info_netstat(route_data):
         routes['ipv4'].append(entry)
 
     try:
-        (route_data6, _err6) = util.subp(
+        (route_data6, _err6) = subp.subp(
             ["netstat", "-A", "inet6", "--route", "--numeric"], rcs=[0, 1])
-    except util.ProcessExecutionError:
+    except subp.ProcessExecutionError:
         pass
     else:
         entries6 = route_data6.splitlines()
@@ -393,13 +394,13 @@ def _netdev_route_info_netstat(route_data):
 
 def route_info():
     routes = {}
-    if util.which('ip'):
+    if subp.which('ip'):
         # Try iproute first of all
-        (iproute_out, _err) = util.subp(["ip", "-o", "route", "list"])
+        (iproute_out, _err) = subp.subp(["ip", "-o", "route", "list"])
         routes = _netdev_route_info_iproute(iproute_out)
-    elif util.which('netstat'):
+    elif subp.which('netstat'):
         # Fall back to net-tools if iproute2 is not present
-        (route_out, _err) = util.subp(
+        (route_out, _err) = subp.subp(
             ["netstat", "--route", "--numeric", "--extend"], rcs=[0, 1])
         routes = _netdev_route_info_netstat(route_out)
     else:
diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index 5270fda8..ac3ecc3d 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -18,9 +18,9 @@ import os.path
 
 from cloudinit import log as logging
 from cloudinit import sources
+from cloudinit import subp
 from cloudinit import util
 
-from cloudinit.util import ProcessExecutionError
 
 LOG = logging.getLogger(__name__)
 
@@ -192,7 +192,7 @@ class DataSourceAltCloud(sources.DataSource):
         # modprobe floppy
         try:
             modprobe_floppy()
-        except ProcessExecutionError as e:
+        except subp.ProcessExecutionError as e:
             util.logexc(LOG, 'Failed modprobe: %s', e)
             return False
 
@@ -201,7 +201,7 @@ class DataSourceAltCloud(sources.DataSource):
         # udevadm settle for floppy device
         try:
             util.udevadm_settle(exists=floppy_dev, timeout=5)
-        except (ProcessExecutionError, OSError) as e:
+        except (subp.ProcessExecutionError, OSError) as e:
             util.logexc(LOG, 'Failed udevadm_settle: %s\n', e)
             return False
 
@@ -261,7 +261,7 @@ class DataSourceAltCloud(sources.DataSource):
 
 
 def modprobe_floppy():
-    out, _err = util.subp(CMD_PROBE_FLOPPY)
+    out, _err = subp.subp(CMD_PROBE_FLOPPY)
     LOG.debug('Command: %s\nOutput%s', ' '.join(CMD_PROBE_FLOPPY), out)
 
 
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 01d9adf2..89312b9e 100755
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -22,6 +22,7 @@ from cloudinit.event import EventType
 from cloudinit.net.dhcp import EphemeralDHCPv4
 from cloudinit import sources
 from cloudinit.sources.helpers import netlink
+from cloudinit import subp
 from cloudinit.url_helper import UrlError, readurl, retry_on_url_exc
 from cloudinit import util
 from cloudinit.reporting import events
@@ -139,8 +140,8 @@ def find_dev_from_busdev(camcontrol_out, busdev):
 
 def execute_or_debug(cmd, fail_ret=None):
     try:
-        return util.subp(cmd)[0]
-    except util.ProcessExecutionError:
+        return subp.subp(cmd)[0]
+    except subp.ProcessExecutionError:
         LOG.debug("Failed to execute: %s", ' '.join(cmd))
         return fail_ret
 
@@ -252,11 +253,11 @@ DEF_PASSWD_REDACTION = 'REDACTED'
 def get_hostname(hostname_command='hostname'):
     if not isinstance(hostname_command, (list, tuple)):
         hostname_command = (hostname_command,)
-    return util.subp(hostname_command, capture=True)[0].strip()
+    return subp.subp(hostname_command, capture=True)[0].strip()
 
 
 def set_hostname(hostname, hostname_command='hostname'):
-    util.subp([hostname_command, hostname])
+    subp.subp([hostname_command, hostname])
 
 
 @azure_ds_telemetry_reporter
@@ -343,7 +344,7 @@ class DataSourceAzure(sources.DataSource):
 
         try:
             invoke_agent(agent_cmd)
-        except util.ProcessExecutionError:
+        except subp.ProcessExecutionError:
             # claim the datasource even if the command failed
             util.logexc(LOG, "agent command '%s' failed.",
                         self.ds_cfg['agent_command'])
@@ -982,7 +983,7 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname):
     if command == "builtin":
         if util.is_FreeBSD():
             command = BOUNCE_COMMAND_FREEBSD
-        elif util.which('ifup'):
+        elif subp.which('ifup'):
             command = BOUNCE_COMMAND_IFUP
         else:
             LOG.debug(
@@ -993,7 +994,7 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname):
     shell = not isinstance(command, (list, tuple))
     # capture=False, see comments in bug 1202758 and bug 1206164.
     util.log_time(logfunc=LOG.debug, msg="publishing hostname",
-                  get_uptime=True, func=util.subp,
+                  get_uptime=True, func=subp.subp,
                   kwargs={'args': command, 'shell': shell, 'capture': False,
                           'env': env})
     return True
@@ -1003,7 +1004,7 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname):
 def crtfile_to_pubkey(fname, data=None):
     pipeline = ('openssl x509 -noout -pubkey < "$0" |'
                 'ssh-keygen -i -m PKCS8 -f /dev/stdin')
-    (out, _err) = util.subp(['sh', '-c', pipeline, fname],
+    (out, _err) = subp.subp(['sh', '-c', pipeline, fname],
                             capture=True, data=data)
     return out.rstrip()
 
@@ -1015,7 +1016,7 @@ def pubkeys_from_crt_files(flist):
     for fname in flist:
         try:
             pubkeys.append(crtfile_to_pubkey(fname))
-        except util.ProcessExecutionError:
+        except subp.ProcessExecutionError:
             errors.append(fname)
 
     if errors:
@@ -1057,7 +1058,7 @@ def invoke_agent(cmd):
     # this is a function itself to simplify patching it for test
     if cmd:
         LOG.debug("invoking agent: %s", cmd)
-        util.subp(cmd, shell=(not isinstance(cmd, list)))
+        subp.subp(cmd, shell=(not isinstance(cmd, list)))
     else:
         LOG.debug("not invoking agent")
 
diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index 2013bed7..54810439 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -22,6 +22,7 @@ from cloudinit import log as logging
 from cloudinit.net import dhcp
 from cloudinit import sources
 from cloudinit import url_helper as uhelp
+from cloudinit import subp
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -46,7 +47,7 @@ class CloudStackPasswordServerClient(object):
         # The password server was in the past, a broken HTTP server, but is now
         # fixed.  wget handles this seamlessly, so it's easier to shell out to
         # that rather than write our own handling code.
-        output, _ = util.subp([
+        output, _ = subp.subp([
             'wget', '--quiet', '--tries', '3', '--timeout', '20',
             '--output-document', '-', '--header',
             'DomU_Request: {0}'.format(domu_request),
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index ae31934b..62756cf7 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -10,6 +10,7 @@ import os
 
 from cloudinit import log as logging
 from cloudinit import sources
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.net import eni
@@ -245,7 +246,7 @@ def find_candidate_devs(probe_optical=True, dslist=None):
         for device in OPTICAL_DEVICES:
             try:
                 util.find_devs_with(path=device)
-            except util.ProcessExecutionError:
+            except subp.ProcessExecutionError:
                 pass
 
     by_fstype = []
diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py
index e0c714e8..d2aa9a58 100644
--- a/cloudinit/sources/DataSourceIBMCloud.py
+++ b/cloudinit/sources/DataSourceIBMCloud.py
@@ -99,6 +99,7 @@ import os
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit.sources.helpers import openstack
+from cloudinit import subp
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -240,7 +241,7 @@ def get_ibm_platform():
     fslabels = {}
     try:
         devs = util.blkid()
-    except util.ProcessExecutionError as e:
+    except subp.ProcessExecutionError as e:
         LOG.warning("Failed to run blkid: %s", e)
         return (None, None)
 
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 41f999e3..4c5eee4f 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -16,6 +16,7 @@ from xml.dom import minidom
 
 from cloudinit import log as logging
 from cloudinit import sources
+from cloudinit import subp
 from cloudinit import util
 from cloudinit.sources.helpers.vmware.imc.config \
     import Config
@@ -536,15 +537,15 @@ def transport_iso9660(require_iso=True):
 def transport_vmware_guestinfo():
     rpctool = "vmware-rpctool"
     not_found = None
-    if not util.which(rpctool):
+    if not subp.which(rpctool):
         return not_found
     cmd = [rpctool, "info-get guestinfo.ovfEnv"]
     try:
-        out, _err = util.subp(cmd)
+        out, _err = subp.subp(cmd)
         if out:
             return out
         LOG.debug("cmd %s exited 0 with empty stdout: %s", cmd, out)
-    except util.ProcessExecutionError as e:
+    except subp.ProcessExecutionError as e:
         if e.exit_code != 1:
             LOG.warning("%s exited with code %d", rpctool, e.exit_code)
             LOG.debug(e)
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index a08ab404..c7fdc2a5 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -21,6 +21,7 @@ import string
 from cloudinit import log as logging
 from cloudinit import net
 from cloudinit import sources
+from cloudinit import subp
 from cloudinit import util
 
 
@@ -334,7 +335,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None,
 
     cmd.extend(bash)
 
-    (output, _error) = util.subp(cmd, data=bcmd)
+    (output, _error) = subp.subp(cmd, data=bcmd)
 
     # exclude vars in bash that change on their own or that we used
     excluded = (
@@ -396,7 +397,7 @@ def read_context_disk_dir(source_dir, asuser=None):
             path = os.path.join(source_dir, 'context.sh')
             content = util.load_file(path)
             context = parse_shell_config(content, asuser=asuser)
-        except util.ProcessExecutionError as e:
+        except subp.ProcessExecutionError as e:
             raise BrokenContextDiskDir("Error processing context.sh: %s" % (e))
         except IOError as e:
             raise NonContextDiskDir("Error reading context.sh: %s" % (e))
diff --git a/cloudinit/sources/DataSourceRbxCloud.py b/cloudinit/sources/DataSourceRbxCloud.py
index 084cb7d5..38fb5421 100644
--- a/cloudinit/sources/DataSourceRbxCloud.py
+++ b/cloudinit/sources/DataSourceRbxCloud.py
@@ -15,6 +15,7 @@ import os.path
 
 from cloudinit import log as logging
 from cloudinit import sources
+from cloudinit import subp
 from cloudinit import util
 from cloudinit.event import EventType
 
@@ -43,11 +44,11 @@ def int2ip(addr):
 
 def _sub_arp(cmd):
     """
-    Uses the prefered cloud-init subprocess def of util.subp
+    Uses the prefered cloud-init subprocess def of subp.subp
     and runs arping.  Breaking this to a separate function
     for later use in mocking and unittests
     """
-    return util.subp(['arping'] + cmd)
+    return subp.subp(['arping'] + cmd)
 
 
 def gratuitous_arp(items, distro):
@@ -61,7 +62,7 @@ def gratuitous_arp(items, distro):
                 source_param, item['source'],
                 item['destination']
             ])
-        except util.ProcessExecutionError as error:
+        except subp.ProcessExecutionError as error:
             # warning, because the system is able to function properly
             # despite no success - some ARP table may be waiting for
             # expiration, but the system may continue
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index cf676504..843b3a2a 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -33,6 +33,7 @@ import socket
 from cloudinit import log as logging
 from cloudinit import serial
 from cloudinit import sources
+from cloudinit import subp
 from cloudinit import util
 from cloudinit.event import EventType
 
@@ -696,9 +697,9 @@ def identify_file(content_f):
     cmd = ["file", "--brief", "--mime-type", content_f]
     f_type = None
     try:
-        (f_type, _err) = util.subp(cmd)
+        (f_type, _err) = subp.subp(cmd)
         LOG.debug("script %s mime type is %s", content_f, f_type)
-    except util.ProcessExecutionError as e:
+    except subp.ProcessExecutionError as e:
         util.logexc(
             LOG, ("Failed to identify script type for %s" % content_f, e))
     return None if f_type is None else f_type.strip()
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 82b6730c..7bace8ca 100755
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -15,6 +15,7 @@ from cloudinit import temp_utils
 from contextlib import contextmanager
 from xml.etree import ElementTree
 
+from cloudinit import subp
 from cloudinit import url_helper
 from cloudinit import util
 from cloudinit import version
@@ -92,7 +93,7 @@ def get_boot_telemetry():
         raise RuntimeError("Failed to determine kernel start timestamp")
 
     try:
-        out, _ = util.subp(['/bin/systemctl',
+        out, _ = subp.subp(['/bin/systemctl',
                             'show', '-p',
                             'UserspaceTimestampMonotonic'],
                            capture=True)
@@ -105,7 +106,7 @@ def get_boot_telemetry():
                                "UserspaceTimestampMonotonic from systemd")
 
         user_start = kernel_start + (float(tsm) / 1000000)
-    except util.ProcessExecutionError as e:
+    except subp.ProcessExecutionError as e:
         raise RuntimeError("Failed to get UserspaceTimestampMonotonic: %s"
                            % e)
     except ValueError as e:
@@ -114,7 +115,7 @@ def get_boot_telemetry():
                            % e)
 
     try:
-        out, _ = util.subp(['/bin/systemctl', 'show',
+        out, _ = subp.subp(['/bin/systemctl', 'show',
                             'cloud-init-local', '-p',
                             'InactiveExitTimestampMonotonic'],
                            capture=True)
@@ -126,7 +127,7 @@ def get_boot_telemetry():
                                "InactiveExitTimestampMonotonic from systemd")
 
         cloudinit_activation = kernel_start + (float(tsm) / 1000000)
-    except util.ProcessExecutionError as e:
+    except subp.ProcessExecutionError as e:
         raise RuntimeError("Failed to get InactiveExitTimestampMonotonic: %s"
                            % e)
     except ValueError as e:
@@ -284,7 +285,7 @@ class OpenSSLManager(object):
             LOG.debug('Certificate already generated.')
             return
         with cd(self.tmpdir):
-            util.subp([
+            subp.subp([
                 'openssl', 'req', '-x509', '-nodes', '-subj',
                 '/CN=LinuxTransport', '-days', '32768', '-newkey', 'rsa:2048',
                 '-keyout', self.certificate_names['private_key'],
@@ -301,14 +302,14 @@ class OpenSSLManager(object):
     @azure_ds_telemetry_reporter
     def _run_x509_action(action, cert):
         cmd = ['openssl', 'x509', '-noout', action]
-        result, _ = util.subp(cmd, data=cert)
+        result, _ = subp.subp(cmd, data=cert)
         return result
 
     @azure_ds_telemetry_reporter
     def _get_ssh_key_from_cert(self, certificate):
         pub_key = self._run_x509_action('-pubkey', certificate)
         keygen_cmd = ['ssh-keygen', '-i', '-m', 'PKCS8', '-f', '/dev/stdin']
-        ssh_key, _ = util.subp(keygen_cmd, data=pub_key)
+        ssh_key, _ = subp.subp(keygen_cmd, data=pub_key)
         return ssh_key
 
     @azure_ds_telemetry_reporter
@@ -341,7 +342,7 @@ class OpenSSLManager(object):
             certificates_content.encode('utf-8'),
         ]
         with cd(self.tmpdir):
-            out, _ = util.subp(
+            out, _ = subp.subp(
                 'openssl cms -decrypt -in /dev/stdin -inkey'
                 ' {private_key} -recip {certificate} | openssl pkcs12 -nodes'
                 ' -password pass:'.format(**self.certificate_names),
diff --git a/cloudinit/sources/helpers/digitalocean.py b/cloudinit/sources/helpers/digitalocean.py
index 0e7cccac..f5bbe46a 100644
--- a/cloudinit/sources/helpers/digitalocean.py
+++ b/cloudinit/sources/helpers/digitalocean.py
@@ -8,6 +8,7 @@ import random
 from cloudinit import log as logging
 from cloudinit import net as cloudnet
 from cloudinit import url_helper
+from cloudinit import subp
 from cloudinit import util
 
 NIC_MAP = {'public': 'eth0', 'private': 'eth1'}
@@ -36,14 +37,14 @@ def assign_ipv4_link_local(nic=None):
     ip_addr_cmd = ['ip', 'addr', 'add', addr, 'dev', nic]
     ip_link_cmd = ['ip', 'link', 'set', 'dev', nic, 'up']
 
-    if not util.which('ip'):
+    if not subp.which('ip'):
         raise RuntimeError("No 'ip' command available to configure ip4LL "
                            "address")
 
     try:
-        util.subp(ip_addr_cmd)
+        subp.subp(ip_addr_cmd)
         LOG.debug("assigned ip4LL address '%s' to '%s'", addr, nic)
-        util.subp(ip_link_cmd)
+        subp.subp(ip_link_cmd)
         LOG.debug("brought device '%s' up", nic)
     except Exception:
         util.logexc(LOG, "ip4LL address assignment of '%s' to '%s' failed."
@@ -74,7 +75,7 @@ def del_ipv4_link_local(nic=None):
     ip_addr_cmd = ['ip', 'addr', 'flush', 'dev', nic]
 
     try:
-        util.subp(ip_addr_cmd)
+        subp.subp(ip_addr_cmd)
         LOG.debug("removed ip4LL addresses from %s", nic)
 
     except Exception as e:
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index a4373f24..c538720a 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -16,6 +16,7 @@ from cloudinit import ec2_utils
 from cloudinit import log as logging
 from cloudinit import net
 from cloudinit import sources
+from cloudinit import subp
 from cloudinit import url_helper
 from cloudinit import util
 from cloudinit.sources import BrokenMetadata
@@ -110,7 +111,7 @@ class SourceMixin(object):
             dev_entries = util.find_devs_with(criteria)
             if dev_entries:
                 device = dev_entries[0]
-        except util.ProcessExecutionError:
+        except subp.ProcessExecutionError:
             pass
         return device
 
diff --git a/cloudinit/sources/helpers/vmware/imc/config_custom_script.py b/cloudinit/sources/helpers/vmware/imc/config_custom_script.py
index 9f14770e..2ab22de9 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_custom_script.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_custom_script.py
@@ -9,6 +9,7 @@ import logging
 import os
 import stat
 
+from cloudinit import subp
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -61,7 +62,7 @@ class PreCustomScript(RunCustomScript):
         """Executing custom script with precustomization argument."""
         LOG.debug("Executing pre-customization script")
         self.prepare_script()
-        util.subp([CustomScriptConstant.CUSTOM_SCRIPT, "precustomization"])
+        subp.subp([CustomScriptConstant.CUSTOM_SCRIPT, "precustomization"])
 
 
 class PostCustomScript(RunCustomScript):
diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py
index 77cbf3b6..3745a262 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_nic.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py
@@ -10,6 +10,7 @@ import os
 import re
 
 from cloudinit.net.network_state import mask_to_net_prefix
+from cloudinit import subp
 from cloudinit import util
 
 logger = logging.getLogger(__name__)
@@ -73,7 +74,7 @@ class NicConfigurator(object):
         The mac address(es) are in the lower case
         """
         cmd = ['ip', 'addr', 'show']
-        output, _err = util.subp(cmd)
+        output, _err = subp.subp(cmd)
         sections = re.split(r'\n\d+: ', '\n' + output)[1:]
 
         macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))'
@@ -248,8 +249,8 @@ class NicConfigurator(object):
         logger.info('Clearing DHCP leases')
 
         # Ignore the return code 1.
-        util.subp(["pkill", "dhclient"], rcs=[0, 1])
-        util.subp(["rm", "-f", "/var/lib/dhcp/*"])
+        subp.subp(["pkill", "dhclient"], rcs=[0, 1])
+        subp.subp(["rm", "-f", "/var/lib/dhcp/*"])
 
     def configure(self, osfamily=None):
         """
diff --git a/cloudinit/sources/helpers/vmware/imc/config_passwd.py b/cloudinit/sources/helpers/vmware/imc/config_passwd.py
index 8c91fa41..d16a7690 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_passwd.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_passwd.py
@@ -9,6 +9,7 @@
 import logging
 import os
 
+from cloudinit import subp
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -56,10 +57,10 @@ class PasswordConfigurator(object):
         LOG.info('Expiring password.')
         for user in uidUserList:
             try:
-                util.subp(['passwd', '--expire', user])
-            except util.ProcessExecutionError as e:
+                subp.subp(['passwd', '--expire', user])
+            except subp.ProcessExecutionError as e:
                 if os.path.exists('/usr/bin/chage'):
-                    util.subp(['chage', '-d', '0', user])
+                    subp.subp(['chage', '-d', '0', user])
                 else:
                     LOG.warning('Failed to expire password for %s with error: '
                                 '%s', user, e)
diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py
index c60a38d7..893b1365 100644
--- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py
+++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py
@@ -10,7 +10,7 @@ import os
 import re
 import time
 
-from cloudinit import util
+from cloudinit import subp
 
 from .guestcust_event import GuestCustEventEnum
 from .guestcust_state import GuestCustStateEnum
@@ -34,7 +34,7 @@ def send_rpc(rpc):
 
     try:
         logger.debug("Sending RPC command: %s", rpc)
-        (out, err) = util.subp(["vmware-rpctool", rpc], rcs=[0])
+        (out, err) = subp.subp(["vmware-rpctool", rpc], rcs=[0])
         # Remove the trailing newline in the output.
         if out:
             out = out.rstrip()
@@ -128,7 +128,7 @@ def get_tools_config(section, key, defaultVal):
                   not installed.
     """
 
-    if not util.which('vmware-toolbox-cmd'):
+    if not subp.which('vmware-toolbox-cmd'):
         logger.debug(
             'vmware-toolbox-cmd not installed, returning default value')
         return defaultVal
@@ -137,7 +137,7 @@ def get_tools_config(section, key, defaultVal):
     cmd = ['vmware-toolbox-cmd', 'config', 'get', section, key]
 
     try:
-        (outText, _) = util.subp(cmd)
+        (outText, _) = subp.subp(cmd)
         m = re.match(r'([^=]+)=(.*)', outText)
         if m:
             retValue = m.group(2).strip()
@@ -147,7 +147,7 @@ def get_tools_config(section, key, defaultVal):
             logger.debug(
                 "Tools config: [%s] %s is not found, return default value: %s",
                 section, key, retValue)
-    except util.ProcessExecutionError as e:
+    except subp.ProcessExecutionError as e:
         logger.error("Failed running %s[%s]", cmd, e.exit_code)
         logger.exception(e)
 
diff --git a/cloudinit/subp.py b/cloudinit/subp.py
index 0ad09306..f8400b1f 100644
--- a/cloudinit/subp.py
+++ b/cloudinit/subp.py
@@ -1,9 +1,11 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 """Common utility functions for interacting with subprocess."""
 
-# TODO move subp shellify and runparts related functions out of util.py
-
 import logging
+import os
+import subprocess
+
+from errno import ENOEXEC
 
 LOG = logging.getLogger(__name__)
 
@@ -54,4 +56,299 @@ def prepend_base_command(base_command, commands):
     return fixed_commands
 
 
+class ProcessExecutionError(IOError):
+
+    MESSAGE_TMPL = ('%(description)s\n'
+                    'Command: %(cmd)s\n'
+                    'Exit code: %(exit_code)s\n'
+                    'Reason: %(reason)s\n'
+                    'Stdout: %(stdout)s\n'
+                    'Stderr: %(stderr)s')
+    empty_attr = '-'
+
+    def __init__(self, stdout=None, stderr=None,
+                 exit_code=None, cmd=None,
+                 description=None, reason=None,
+                 errno=None):
+        if not cmd:
+            self.cmd = self.empty_attr
+        else:
+            self.cmd = cmd
+
+        if not description:
+            if not exit_code and errno == ENOEXEC:
+                self.description = 'Exec format error. Missing #! in script?'
+            else:
+                self.description = 'Unexpected error while running command.'
+        else:
+            self.description = description
+
+        if not isinstance(exit_code, int):
+            self.exit_code = self.empty_attr
+        else:
+            self.exit_code = exit_code
+
+        if not stderr:
+            if stderr is None:
+                self.stderr = self.empty_attr
+            else:
+                self.stderr = stderr
+        else:
+            self.stderr = self._indent_text(stderr)
+
+        if not stdout:
+            if stdout is None:
+                self.stdout = self.empty_attr
+            else:
+                self.stdout = stdout
+        else:
+            self.stdout = self._indent_text(stdout)
+
+        if reason:
+            self.reason = reason
+        else:
+            self.reason = self.empty_attr
+
+        self.errno = errno
+        message = self.MESSAGE_TMPL % {
+            'description': self._ensure_string(self.description),
+            'cmd': self._ensure_string(self.cmd),
+            'exit_code': self._ensure_string(self.exit_code),
+            'stdout': self._ensure_string(self.stdout),
+            'stderr': self._ensure_string(self.stderr),
+            'reason': self._ensure_string(self.reason),
+        }
+        IOError.__init__(self, message)
+
+    def _ensure_string(self, text):
+        """
+        if data is bytes object, decode
+        """
+        return text.decode() if isinstance(text, bytes) else text
+
+    def _indent_text(self, text, indent_level=8):
+        """
+        indent text on all but the first line, allowing for easy to read output
+        """
+        cr = '\n'
+        indent = ' ' * indent_level
+        # if input is bytes, return bytes
+        if isinstance(text, bytes):
+            cr = cr.encode()
+            indent = indent.encode()
+        # remove any newlines at end of text first to prevent unneeded blank
+        # line in output
+        return text.rstrip(cr).replace(cr, cr + indent)
+
+
+def subp(args, data=None, rcs=None, env=None, capture=True,
+         combine_capture=False, shell=False,
+         logstring=False, decode="replace", target=None, update_env=None,
+         status_cb=None):
+    """Run a subprocess.
+
+    :param args: command to run in a list. [cmd, arg1, arg2...]
+    :param data: input to the command, made available on its stdin.
+    :param rcs:
+        a list of allowed return codes.  If subprocess exits with a value not
+        in this list, a ProcessExecutionError will be raised.  By default,
+        data is returned as a string.  See 'decode' parameter.
+    :param env: a dictionary for the command's environment.
+    :param capture:
+        boolean indicating if output should be captured.  If True, then stderr
+        and stdout will be returned.  If False, they will not be redirected.
+    :param combine_capture:
+        boolean indicating if stderr should be redirected to stdout. When True,
+        interleaved stderr and stdout will be returned as the first element of
+        a tuple, the second will be empty string or bytes (per decode).
+        if combine_capture is True, then output is captured independent of
+        the value of capture.
+    :param shell: boolean indicating if this should be run with a shell.
+    :param logstring:
+        the command will be logged to DEBUG.  If it contains info that should
+        not be logged, then logstring will be logged instead.
+    :param decode:
+        if False, no decoding will be done and returned stdout and stderr will
+        be bytes.  Other allowed values are 'strict', 'ignore', and 'replace'.
+        These values are passed through to bytes().decode() as the 'errors'
+        parameter.  There is no support for decoding to other than utf-8.
+    :param target:
+        not supported, kwarg present only to make function signature similar
+        to curtin's subp.
+    :param update_env:
+        update the enviornment for this command with this dictionary.
+        this will not affect the current processes os.environ.
+    :param status_cb:
+        call this fuction with a single string argument before starting
+        and after finishing.
+
+    :return
+        if not capturing, return is (None, None)
+        if capturing, stdout and stderr are returned.
+            if decode:
+                entries in tuple will be python2 unicode or python3 string
+            if not decode:
+                entries in tuple will be python2 string or python3 bytes
+    """
+
+    # not supported in cloud-init (yet), for now kept in the call signature
+    # to ease maintaining code shared between cloud-init and curtin
+    if target is not None:
+        raise ValueError("target arg not supported by cloud-init")
+
+    if rcs is None:
+        rcs = [0]
+
+    devnull_fp = None
+
+    if update_env:
+        if env is None:
+            env = os.environ
+        env = env.copy()
+        env.update(update_env)
+
+    if target_path(target) != "/":
+        args = ['chroot', target] + list(args)
+
+    if status_cb:
+        command = ' '.join(args) if isinstance(args, list) else args
+        status_cb('Begin run command: {command}\n'.format(command=command))
+    if not logstring:
+        LOG.debug(("Running command %s with allowed return codes %s"
+                   " (shell=%s, capture=%s)"),
+                  args, rcs, shell, 'combine' if combine_capture else capture)
+    else:
+        LOG.debug(("Running hidden command to protect sensitive "
+                   "input/output logstring: %s"), logstring)
+
+    stdin = None
+    stdout = None
+    stderr = None
+    if capture:
+        stdout = subprocess.PIPE
+        stderr = subprocess.PIPE
+    if combine_capture:
+        stdout = subprocess.PIPE
+        stderr = subprocess.STDOUT
+    if data is None:
+        # using devnull assures any reads get null, rather
+        # than possibly waiting on input.
+        devnull_fp = open(os.devnull)
+        stdin = devnull_fp
+    else:
+        stdin = subprocess.PIPE
+        if not isinstance(data, bytes):
+            data = data.encode()
+
+    # Popen converts entries in the arguments array from non-bytes to bytes.
+    # When locale is unset it may use ascii for that encoding which can
+    # cause UnicodeDecodeErrors. (LP: #1751051)
+    if isinstance(args, bytes):
+        bytes_args = args
+    elif isinstance(args, str):
+        bytes_args = args.encode("utf-8")
+    else:
+        bytes_args = [
+            x if isinstance(x, bytes) else x.encode("utf-8")
+            for x in args]
+    try:
+        sp = subprocess.Popen(bytes_args, stdout=stdout,
+                              stderr=stderr, stdin=stdin,
+                              env=env, shell=shell)
+        (out, err) = sp.communicate(data)
+    except OSError as e:
+        if status_cb:
+            status_cb('ERROR: End run command: invalid command provided\n')
+        raise ProcessExecutionError(
+            cmd=args, reason=e, errno=e.errno,
+            stdout="-" if decode else b"-",
+            stderr="-" if decode else b"-")
+    finally:
+        if devnull_fp:
+            devnull_fp.close()
+
+    # Just ensure blank instead of none.
+    if capture or combine_capture:
+        if not out:
+            out = b''
+        if not err:
+            err = b''
+    if decode:
+        def ldecode(data, m='utf-8'):
+            if not isinstance(data, bytes):
+                return data
+            return data.decode(m, decode)
+
+        out = ldecode(out)
+        err = ldecode(err)
+
+    rc = sp.returncode
+    if rc not in rcs:
+        if status_cb:
+            status_cb(
+                'ERROR: End run command: exit({code})\n'.format(code=rc))
+        raise ProcessExecutionError(stdout=out, stderr=err,
+                                    exit_code=rc,
+                                    cmd=args)
+    if status_cb:
+        status_cb('End run command: exit({code})\n'.format(code=rc))
+    return (out, err)
+
+
+def target_path(target, path=None):
+    # return 'path' inside target, accepting target as None
+    if target in (None, ""):
+        target = "/"
+    elif not isinstance(target, str):
+        raise ValueError("Unexpected input for target: %s" % target)
+    else:
+        target = os.path.abspath(target)
+        # abspath("//") returns "//" specifically for 2 slashes.
+        if target.startswith("//"):
+            target = target[1:]
+
+    if not path:
+        return target
+
+    # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /.
+    while len(path) and path[0] == "/":
+        path = path[1:]
+
+    return os.path.join(target, path)
+
+
+def which(program, search=None, target=None):
+    target = target_path(target)
+
+    if os.path.sep in program:
+        # if program had a '/' in it, then do not search PATH
+        # 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls
+        # so effectively we set cwd to / (or target)
+        if is_exe(target_path(target, program)):
+            return program
+
+    if search is None:
+        paths = [p.strip('"') for p in
+                 os.environ.get("PATH", "").split(os.pathsep)]
+        if target == "/":
+            search = paths
+        else:
+            search = [p for p in paths if p.startswith("/")]
+
+    # normalize path input
+    search = [os.path.abspath(p) for p in search]
+
+    for path in search:
+        ppath = os.path.sep.join((path, program))
+        if is_exe(target_path(target, ppath)):
+            return ppath
+
+    return None
+
+
+def is_exe(fpath):
+    # return boolean indicating if fpath exists and is executable.
+    return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
index f3ab7e8c..58f63b69 100644
--- a/cloudinit/tests/helpers.py
+++ b/cloudinit/tests/helpers.py
@@ -23,9 +23,10 @@ from cloudinit import distros
 from cloudinit import helpers as ch
 from cloudinit.sources import DataSourceNone
 from cloudinit.templater import JINJA_AVAILABLE
+from cloudinit import subp
 from cloudinit import util
 
-_real_subp = util.subp
+_real_subp = subp.subp
 
 # Used for skipping tests
 SkipTest = unittest.SkipTest
@@ -134,9 +135,9 @@ class CiTestCase(TestCase):
             self.old_handlers = self.logger.handlers
             self.logger.handlers = [handler]
         if self.allowed_subp is True:
-            util.subp = _real_subp
+            subp.subp = _real_subp
         else:
-            util.subp = self._fake_subp
+            subp.subp = self._fake_subp
 
     def _fake_subp(self, *args, **kwargs):
         if 'args' in kwargs:
@@ -171,7 +172,7 @@ class CiTestCase(TestCase):
             # Remove the handler we setup
             logging.getLogger().handlers = self.old_handlers
             logging.getLogger().level = None
-        util.subp = _real_subp
+        subp.subp = _real_subp
         super(CiTestCase, self).tearDown()
 
     def tmp_dir(self, dir=None, cleanup=True):
@@ -280,13 +281,13 @@ class FilesystemMockingTestCase(ResourceUsingTestCase):
                     mock.patch.object(mod, f, trap_func))
 
         # Handle subprocess calls
-        func = getattr(util, 'subp')
+        func = getattr(subp, 'subp')
 
         def nsubp(*_args, **_kwargs):
             return ('', '')
 
         self.patched_funcs.enter_context(
-            mock.patch.object(util, 'subp', nsubp))
+            mock.patch.object(subp, 'subp', nsubp))
 
         def null_func(*_args, **_kwargs):
             return None
diff --git a/cloudinit/tests/test_conftest.py b/cloudinit/tests/test_conftest.py
index 773ef8fe..a6537248 100644
--- a/cloudinit/tests/test_conftest.py
+++ b/cloudinit/tests/test_conftest.py
@@ -1,6 +1,6 @@
 import pytest
 
-from cloudinit import util
+from cloudinit import subp
 from cloudinit.tests.helpers import CiTestCase
 
 
@@ -9,17 +9,17 @@ class TestDisableSubpUsage:
 
     def test_using_subp_raises_assertion_error(self):
         with pytest.raises(AssertionError):
-            util.subp(["some", "args"])
+            subp.subp(["some", "args"])
 
     def test_typeerrors_on_incorrect_usage(self):
         with pytest.raises(TypeError):
             # We are intentionally passing no value for a parameter, so:
             #  pylint: disable=no-value-for-parameter
-            util.subp()
+            subp.subp()
 
     @pytest.mark.parametrize('disable_subp_usage', [False], indirect=True)
     def test_subp_usage_can_be_reenabled(self):
-        util.subp(['whoami'])
+        subp.subp(['whoami'])
 
     @pytest.mark.parametrize(
         'disable_subp_usage', [['whoami'], 'whoami'], indirect=True)
@@ -27,18 +27,18 @@ class TestDisableSubpUsage:
         # The two parameters test each potential invocation with a single
         # argument
         with pytest.raises(AssertionError) as excinfo:
-            util.subp(["some", "args"])
+            subp.subp(["some", "args"])
         assert "allowed: whoami" in str(excinfo.value)
-        util.subp(['whoami'])
+        subp.subp(['whoami'])
 
     @pytest.mark.parametrize(
         'disable_subp_usage', [['whoami', 'bash']], indirect=True)
     def test_subp_usage_can_be_conditionally_reenabled_for_multiple_cmds(self):
         with pytest.raises(AssertionError) as excinfo:
-            util.subp(["some", "args"])
+            subp.subp(["some", "args"])
         assert "allowed: whoami,bash" in str(excinfo.value)
-        util.subp(['bash', '-c', 'true'])
-        util.subp(['whoami'])
+        subp.subp(['bash', '-c', 'true'])
+        subp.subp(['whoami'])
 
 
 class TestDisableSubpUsageInTestSubclass(CiTestCase):
@@ -46,16 +46,16 @@ class TestDisableSubpUsageInTestSubclass(CiTestCase):
 
     def test_using_subp_raises_exception(self):
         with pytest.raises(Exception):
-            util.subp(["some", "args"])
+            subp.subp(["some", "args"])
 
     def test_typeerrors_on_incorrect_usage(self):
         with pytest.raises(TypeError):
-            util.subp()
+            subp.subp()
 
     def test_subp_usage_can_be_reenabled(self):
         _old_allowed_subp = self.allow_subp
         self.allowed_subp = True
         try:
-            util.subp(['bash', '-c', 'true'])
+            subp.subp(['bash', '-c', 'true'])
         finally:
             self.allowed_subp = _old_allowed_subp
diff --git a/cloudinit/tests/test_gpg.py b/cloudinit/tests/test_gpg.py
index 8dd57137..f96f5372 100644
--- a/cloudinit/tests/test_gpg.py
+++ b/cloudinit/tests/test_gpg.py
@@ -4,19 +4,19 @@
 from unittest import mock
 
 from cloudinit import gpg
-from cloudinit import util
+from cloudinit import subp
 from cloudinit.tests.helpers import CiTestCase
 
 
 @mock.patch("cloudinit.gpg.time.sleep")
-@mock.patch("cloudinit.gpg.util.subp")
+@mock.patch("cloudinit.gpg.subp.subp")
 class TestReceiveKeys(CiTestCase):
     """Test the recv_key method."""
 
     def test_retries_on_subp_exc(self, m_subp, m_sleep):
         """retry should be done on gpg receive keys failure."""
         retries = (1, 2, 4)
-        my_exc = util.ProcessExecutionError(
+        my_exc = subp.ProcessExecutionError(
             stdout='', stderr='', exit_code=2, cmd=['mycmd'])
         m_subp.side_effect = (my_exc, my_exc, ('', ''))
         gpg.recv_key("ABCD", "keyserver.example.com", retries=retries)
@@ -26,7 +26,7 @@ class TestReceiveKeys(CiTestCase):
         """If the final run fails, error should be raised."""
         naplen = 1
         keyid, keyserver = ("ABCD", "keyserver.example.com")
-        m_subp.side_effect = util.ProcessExecutionError(
+        m_subp.side_effect = subp.ProcessExecutionError(
             stdout='', stderr='', exit_code=2, cmd=['mycmd'])
         with self.assertRaises(ValueError) as rcm:
             gpg.recv_key(keyid, keyserver, retries=(naplen,))
@@ -36,7 +36,7 @@ class TestReceiveKeys(CiTestCase):
 
     def test_no_retries_on_none(self, m_subp, m_sleep):
         """retry should not be done if retries is None."""
-        m_subp.side_effect = util.ProcessExecutionError(
+        m_subp.side_effect = subp.ProcessExecutionError(
             stdout='', stderr='', exit_code=2, cmd=['mycmd'])
         with self.assertRaises(ValueError):
             gpg.recv_key("ABCD", "keyserver.example.com", retries=None)
diff --git a/cloudinit/tests/test_netinfo.py b/cloudinit/tests/test_netinfo.py
index 1c8a791e..e44b16d8 100644
--- a/cloudinit/tests/test_netinfo.py
+++ b/cloudinit/tests/test_netinfo.py
@@ -27,8 +27,8 @@ class TestNetInfo(CiTestCase):
     maxDiff = None
     with_logs = True
 
-    @mock.patch('cloudinit.netinfo.util.which')
-    @mock.patch('cloudinit.netinfo.util.subp')
+    @mock.patch('cloudinit.netinfo.subp.which')
+    @mock.patch('cloudinit.netinfo.subp.subp')
     def test_netdev_old_nettools_pformat(self, m_subp, m_which):
         """netdev_pformat properly rendering old nettools info."""
         m_subp.return_value = (SAMPLE_OLD_IFCONFIG_OUT, '')
@@ -36,8 +36,8 @@ class TestNetInfo(CiTestCase):
         content = netdev_pformat()
         self.assertEqual(NETDEV_FORMATTED_OUT, content)
 
-    @mock.patch('cloudinit.netinfo.util.which')
-    @mock.patch('cloudinit.netinfo.util.subp')
+    @mock.patch('cloudinit.netinfo.subp.which')
+    @mock.patch('cloudinit.netinfo.subp.subp')
     def test_netdev_new_nettools_pformat(self, m_subp, m_which):
         """netdev_pformat properly rendering netdev new nettools info."""
         m_subp.return_value = (SAMPLE_NEW_IFCONFIG_OUT, '')
@@ -45,8 +45,8 @@ class TestNetInfo(CiTestCase):
         content = netdev_pformat()
         self.assertEqual(NETDEV_FORMATTED_OUT, content)
 
-    @mock.patch('cloudinit.netinfo.util.which')
-    @mock.patch('cloudinit.netinfo.util.subp')
+    @mock.patch('cloudinit.netinfo.subp.which')
+    @mock.patch('cloudinit.netinfo.subp.subp')
     def test_netdev_freebsd_nettools_pformat(self, m_subp, m_which):
         """netdev_pformat properly rendering netdev new nettools info."""
         m_subp.return_value = (SAMPLE_FREEBSD_IFCONFIG_OUT, '')
@@ -57,8 +57,8 @@ class TestNetInfo(CiTestCase):
         print()
         self.assertEqual(FREEBSD_NETDEV_OUT, content)
 
-    @mock.patch('cloudinit.netinfo.util.which')
-    @mock.patch('cloudinit.netinfo.util.subp')
+    @mock.patch('cloudinit.netinfo.subp.which')
+    @mock.patch('cloudinit.netinfo.subp.subp')
     def test_netdev_iproute_pformat(self, m_subp, m_which):
         """netdev_pformat properly rendering ip route info."""
         m_subp.return_value = (SAMPLE_IPADDRSHOW_OUT, '')
@@ -72,8 +72,8 @@ class TestNetInfo(CiTestCase):
             '255.0.0.0   |   .    |', '255.0.0.0   |  host  |')
         self.assertEqual(new_output, content)
 
-    @mock.patch('cloudinit.netinfo.util.which')
-    @mock.patch('cloudinit.netinfo.util.subp')
+    @mock.patch('cloudinit.netinfo.subp.which')
+    @mock.patch('cloudinit.netinfo.subp.subp')
     def test_netdev_warn_on_missing_commands(self, m_subp, m_which):
         """netdev_pformat warns when missing both ip and 'netstat'."""
         m_which.return_value = None  # Niether ip nor netstat found
@@ -85,8 +85,8 @@ class TestNetInfo(CiTestCase):
             self.logs.getvalue())
         m_subp.assert_not_called()
 
-    @mock.patch('cloudinit.netinfo.util.which')
-    @mock.patch('cloudinit.netinfo.util.subp')
+    @mock.patch('cloudinit.netinfo.subp.which')
+    @mock.patch('cloudinit.netinfo.subp.subp')
     def test_netdev_info_nettools_down(self, m_subp, m_which):
         """test netdev_info using nettools and down interfaces."""
         m_subp.return_value = (
@@ -100,8 +100,8 @@ class TestNetInfo(CiTestCase):
                     'hwaddr': '.', 'up': True}},
             netdev_info("."))
 
-    @mock.patch('cloudinit.netinfo.util.which')
-    @mock.patch('cloudinit.netinfo.util.subp')
+    @mock.patch('cloudinit.netinfo.subp.which')
+    @mock.patch('cloudinit.netinfo.subp.subp')
     def test_netdev_info_iproute_down(self, m_subp, m_which):
         """Test netdev_info with ip and down interfaces."""
         m_subp.return_value = (
@@ -130,8 +130,8 @@ class TestNetInfo(CiTestCase):
             readResource("netinfo/netdev-formatted-output-down"),
             netdev_pformat())
 
-    @mock.patch('cloudinit.netinfo.util.which')
-    @mock.patch('cloudinit.netinfo.util.subp')
+    @mock.patch('cloudinit.netinfo.subp.which')
+    @mock.patch('cloudinit.netinfo.subp.subp')
     def test_route_nettools_pformat(self, m_subp, m_which):
         """route_pformat properly rendering nettools route info."""
 
@@ -147,8 +147,8 @@ class TestNetInfo(CiTestCase):
         content = route_pformat()
         self.assertEqual(ROUTE_FORMATTED_OUT, content)
 
-    @mock.patch('cloudinit.netinfo.util.which')
-    @mock.patch('cloudinit.netinfo.util.subp')
+    @mock.patch('cloudinit.netinfo.subp.which')
+    @mock.patch('cloudinit.netinfo.subp.subp')
     def test_route_iproute_pformat(self, m_subp, m_which):
         """route_pformat properly rendering ip route info."""
 
@@ -165,8 +165,8 @@ class TestNetInfo(CiTestCase):
         content = route_pformat()
         self.assertEqual(ROUTE_FORMATTED_OUT, content)
 
-    @mock.patch('cloudinit.netinfo.util.which')
-    @mock.patch('cloudinit.netinfo.util.subp')
+    @mock.patch('cloudinit.netinfo.subp.which')
+    @mock.patch('cloudinit.netinfo.subp.subp')
     def test_route_warn_on_missing_commands(self, m_subp, m_which):
         """route_pformat warns when missing both ip and 'netstat'."""
         m_which.return_value = None  # Niether ip nor netstat found
diff --git a/cloudinit/tests/test_subp.py b/cloudinit/tests/test_subp.py
index 448097d3..911c1f3d 100644
--- a/cloudinit/tests/test_subp.py
+++ b/cloudinit/tests/test_subp.py
@@ -2,10 +2,21 @@
 
 """Tests for cloudinit.subp utility functions"""
 
-from cloudinit import subp
+import json
+import os
+import sys
+import stat
+
+from unittest import mock
+
+from cloudinit import subp, util
 from cloudinit.tests.helpers import CiTestCase
 
 
+BASH = subp.which('bash')
+BOGUS_COMMAND = 'this-is-not-expected-to-be-a-program-name'
+
+
 class TestPrependBaseCommands(CiTestCase):
 
     with_logs = True
@@ -58,4 +69,218 @@ class TestPrependBaseCommands(CiTestCase):
         self.assertEqual('', self.logs.getvalue())
         self.assertEqual(expected, fixed_commands)
 
+
+class TestSubp(CiTestCase):
+    allowed_subp = [BASH, 'cat', CiTestCase.SUBP_SHELL_TRUE,
+                    BOGUS_COMMAND, sys.executable]
+
+    stdin2err = [BASH, '-c', 'cat >&2']
+    stdin2out = ['cat']
+    utf8_invalid = b'ab\xaadef'
+    utf8_valid = b'start \xc3\xa9 end'
+    utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7'
+    printenv = [BASH, '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--']
+
+    def printf_cmd(self, *args):
+        # bash's printf supports \xaa.  So does /usr/bin/printf
+        # but by using bash, we remove dependency on another program.
+        return([BASH, '-c', 'printf "$@"', 'printf'] + list(args))
+
+    def test_subp_handles_bytestrings(self):
+        """subp can run a bytestring command if shell is True."""
+        tmp_file = self.tmp_path('test.out')
+        cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file)
+        (out, _err) = subp.subp(cmd.encode('utf-8'), shell=True)
+        self.assertEqual(u'', out)
+        self.assertEqual(u'', _err)
+        self.assertEqual('HI MOM\n', util.load_file(tmp_file))
+
+    def test_subp_handles_strings(self):
+        """subp can run a string command if shell is True."""
+        tmp_file = self.tmp_path('test.out')
+        cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file)
+        (out, _err) = subp.subp(cmd, shell=True)
+        self.assertEqual(u'', out)
+        self.assertEqual(u'', _err)
+        self.assertEqual('HI MOM\n', util.load_file(tmp_file))
+
+    def test_subp_handles_utf8(self):
+        # The given bytes contain utf-8 accented characters as seen in e.g.
+        # the "deja dup" package in Ubuntu.
+        cmd = self.printf_cmd(self.utf8_valid_2)
+        (out, _err) = subp.subp(cmd, capture=True)
+        self.assertEqual(out, self.utf8_valid_2.decode('utf-8'))
+
+    def test_subp_respects_decode_false(self):
+        (out, err) = subp.subp(self.stdin2out, capture=True, decode=False,
+                               data=self.utf8_valid)
+        self.assertTrue(isinstance(out, bytes))
+        self.assertTrue(isinstance(err, bytes))
+        self.assertEqual(out, self.utf8_valid)
+
+    def test_subp_decode_ignore(self):
+        # this executes a string that writes invalid utf-8 to stdout
+        (out, _err) = subp.subp(self.printf_cmd('abc\\xaadef'),
+                                capture=True, decode='ignore')
+        self.assertEqual(out, 'abcdef')
+
+    def test_subp_decode_strict_valid_utf8(self):
+        (out, _err) = subp.subp(self.stdin2out, capture=True,
+                                decode='strict', data=self.utf8_valid)
+        self.assertEqual(out, self.utf8_valid.decode('utf-8'))
+
+    def test_subp_decode_invalid_utf8_replaces(self):
+        (out, _err) = subp.subp(self.stdin2out, capture=True,
+                                data=self.utf8_invalid)
+        expected = self.utf8_invalid.decode('utf-8', 'replace')
+        self.assertEqual(out, expected)
+
+    def test_subp_decode_strict_raises(self):
+        args = []
+        kwargs = {'args': self.stdin2out, 'capture': True,
+                  'decode': 'strict', 'data': self.utf8_invalid}
+        self.assertRaises(UnicodeDecodeError, subp.subp, *args, **kwargs)
+
+    def test_subp_capture_stderr(self):
+        data = b'hello world'
+        (out, err) = subp.subp(self.stdin2err, capture=True,
+                               decode=False, data=data,
+                               update_env={'LC_ALL': 'C'})
+        self.assertEqual(err, data)
+        self.assertEqual(out, b'')
+
+    def test_subp_reads_env(self):
+        with mock.patch.dict("os.environ", values={'FOO': 'BAR'}):
+            out, _err = subp.subp(self.printenv + ['FOO'], capture=True)
+        self.assertEqual('FOO=BAR', out.splitlines()[0])
+
+    def test_subp_env_and_update_env(self):
+        out, _err = subp.subp(
+            self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True,
+            env={'FOO': 'BAR'},
+            update_env={'HOME': '/myhome', 'K2': 'V2'})
+        self.assertEqual(
+            ['FOO=BAR', 'HOME=/myhome', 'K1=', 'K2=V2'], out.splitlines())
+
+    def test_subp_update_env(self):
+        extra = {'FOO': 'BAR', 'HOME': '/root', 'K1': 'V1'}
+        with mock.patch.dict("os.environ", values=extra):
+            out, _err = subp.subp(
+                self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True,
+                update_env={'HOME': '/myhome', 'K2': 'V2'})
+
+        self.assertEqual(
+            ['FOO=BAR', 'HOME=/myhome', 'K1=V1', 'K2=V2'], out.splitlines())
+
+    def test_subp_warn_missing_shebang(self):
+        """Warn on no #! in script"""
+        noshebang = self.tmp_path('noshebang')
+        util.write_file(noshebang, 'true\n')
+
+        print("os is %s" % os)
+        os.chmod(noshebang, os.stat(noshebang).st_mode | stat.S_IEXEC)
+        with self.allow_subp([noshebang]):
+            self.assertRaisesRegex(subp.ProcessExecutionError,
+                                   r'Missing #! in script\?',
+                                   subp.subp, (noshebang,))
+
+    def test_subp_combined_stderr_stdout(self):
+        """Providing combine_capture as True redirects stderr to stdout."""
+        data = b'hello world'
+        (out, err) = subp.subp(self.stdin2err, capture=True,
+                               combine_capture=True, decode=False, data=data)
+        self.assertEqual(b'', err)
+        self.assertEqual(data, out)
+
+    def test_returns_none_if_no_capture(self):
+        (out, err) = subp.subp(self.stdin2out, data=b'', capture=False)
+        self.assertIsNone(err)
+        self.assertIsNone(out)
+
+    def test_exception_has_out_err_are_bytes_if_decode_false(self):
+        """Raised exc should have stderr, stdout as bytes if no decode."""
+        with self.assertRaises(subp.ProcessExecutionError) as cm:
+            subp.subp([BOGUS_COMMAND], decode=False)
+        self.assertTrue(isinstance(cm.exception.stdout, bytes))
+        self.assertTrue(isinstance(cm.exception.stderr, bytes))
+
+    def test_exception_has_out_err_are_bytes_if_decode_true(self):
+        """Raised exc should have stderr, stdout as string if no decode."""
+        with self.assertRaises(subp.ProcessExecutionError) as cm:
+            subp.subp([BOGUS_COMMAND], decode=True)
+        self.assertTrue(isinstance(cm.exception.stdout, str))
+        self.assertTrue(isinstance(cm.exception.stderr, str))
+
+    def test_bunch_of_slashes_in_path(self):
+        self.assertEqual("/target/my/path/",
+                         subp.target_path("/target/", "//my/path/"))
+        self.assertEqual("/target/my/path/",
+                         subp.target_path("/target/", "///my/path/"))
+
+    def test_c_lang_can_take_utf8_args(self):
+        """Independent of system LC_CTYPE, args can contain utf-8 strings.
+
+        When python starts up, its default encoding gets set based on
+        the value of LC_CTYPE.  If no system locale is set, the default
+        encoding for both python2 and python3 in some paths will end up
+        being ascii.
+
+        Attempts to use setlocale or patching (or changing) os.environ
+        in the current environment seem to not be effective.
+
+        This test starts up a python with LC_CTYPE set to C so that
+        the default encoding will be set to ascii.  In such an environment
+        Popen(['command', 'non-ascii-arg']) would cause a UnicodeDecodeError.
+        """
+        python_prog = '\n'.join([
+            'import json, sys',
+            'from cloudinit.subp import subp',
+            'data = sys.stdin.read()',
+            'cmd = json.loads(data)',
+            'subp(cmd, capture=False)',
+            ''])
+        cmd = [BASH, '-c', 'echo -n "$@"', '--',
+               self.utf8_valid.decode("utf-8")]
+        python_subp = [sys.executable, '-c', python_prog]
+
+        out, _err = subp.subp(
+            python_subp, update_env={'LC_CTYPE': 'C'},
+            data=json.dumps(cmd).encode("utf-8"),
+            decode=False)
+        self.assertEqual(self.utf8_valid, out)
+
+    def test_bogus_command_logs_status_messages(self):
+        """status_cb gets status messages logs on bogus commands provided."""
+        logs = []
+
+        def status_cb(log):
+            logs.append(log)
+
+        with self.assertRaises(subp.ProcessExecutionError):
+            subp.subp([BOGUS_COMMAND], status_cb=status_cb)
+
+        expected = [
+            'Begin run command: {cmd}\n'.format(cmd=BOGUS_COMMAND),
+            'ERROR: End run command: invalid command provided\n']
+        self.assertEqual(expected, logs)
+
+    def test_command_logs_exit_codes_to_status_cb(self):
+        """status_cb gets status messages containing command exit code."""
+        logs = []
+
+        def status_cb(log):
+            logs.append(log)
+
+        with self.assertRaises(subp.ProcessExecutionError):
+            subp.subp([BASH, '-c', 'exit 2'], status_cb=status_cb)
+        subp.subp([BASH, '-c', 'exit 0'], status_cb=status_cb)
+
+        expected = [
+            'Begin run command: %s -c exit 2\n' % BASH,
+            'ERROR: End run command: exit(2)\n',
+            'Begin run command: %s -c exit 0\n' % BASH,
+            'End run command: exit(0)\n']
+        self.assertEqual(expected, logs)
+
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py
index bfccfe1e..0d9a8fd9 100644
--- a/cloudinit/tests/test_util.py
+++ b/cloudinit/tests/test_util.py
@@ -9,6 +9,7 @@ import platform
 import pytest
 
 import cloudinit.util as util
+from cloudinit import subp
 
 from cloudinit.tests.helpers import CiTestCase, mock
 from textwrap import dedent
@@ -332,7 +333,7 @@ class TestBlkid(CiTestCase):
                           "PARTUUID": self.ids["id09"]},
         })
 
-    @mock.patch("cloudinit.util.subp")
+    @mock.patch("cloudinit.subp.subp")
     def test_functional_blkid(self, m_subp):
         m_subp.return_value = (
             self.blkid_out.format(**self.ids), "")
@@ -340,7 +341,7 @@ class TestBlkid(CiTestCase):
         m_subp.assert_called_with(["blkid", "-o", "full"], capture=True,
                                   decode="replace")
 
-    @mock.patch("cloudinit.util.subp")
+    @mock.patch("cloudinit.subp.subp")
     def test_blkid_no_cache_uses_no_cache(self, m_subp):
         """blkid should turn off cache if disable_cache is true."""
         m_subp.return_value = (
@@ -351,7 +352,7 @@ class TestBlkid(CiTestCase):
                                   capture=True, decode="replace")
 
 
-@mock.patch('cloudinit.util.subp')
+@mock.patch('cloudinit.subp.subp')
 class TestUdevadmSettle(CiTestCase):
     def test_with_no_params(self, m_subp):
         """called with no parameters."""
@@ -396,8 +397,8 @@ class TestUdevadmSettle(CiTestCase):
              '--timeout=%s' % timeout])
 
     def test_subp_exception_raises_to_caller(self, m_subp):
-        m_subp.side_effect = util.ProcessExecutionError("BOOM")
-        self.assertRaises(util.ProcessExecutionError, util.udevadm_settle)
+        m_subp.side_effect = subp.ProcessExecutionError("BOOM")
+        self.assertRaises(subp.ProcessExecutionError, util.udevadm_settle)
 
 
 @mock.patch('os.path.exists')
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 985e7d20..445e3d4c 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -32,12 +32,13 @@ import subprocess
 import sys
 import time
 from base64 import b64decode, b64encode
-from errno import ENOENT, ENOEXEC
+from errno import ENOENT
 from functools import lru_cache
 from urllib import parse
 
 from cloudinit import importer
 from cloudinit import log as logging
+from cloudinit import subp
 from cloudinit import (
     mergers,
     safeyaml,
@@ -74,8 +75,8 @@ def get_dpkg_architecture(target=None):
     N.B. This function is wrapped in functools.lru_cache, so repeated calls
     won't shell out every time.
     """
-    out, _ = subp(['dpkg', '--print-architecture'], capture=True,
-                  target=target)
+    out, _ = subp.subp(['dpkg', '--print-architecture'], capture=True,
+                       target=target)
     return out.strip()
 
 
@@ -86,7 +87,8 @@ def lsb_release(target=None):
 
     data = {}
     try:
-        out, _ = subp(['lsb_release', '--all'], capture=True, target=target)
+        out, _ = subp.subp(['lsb_release', '--all'], capture=True,
+                           target=target)
         for line in out.splitlines():
             fname, _, val = line.partition(":")
             if fname in fmap:
@@ -96,35 +98,13 @@ def lsb_release(target=None):
             LOG.warning("Missing fields in lsb_release --all output: %s",
                         ','.join(missing))
 
-    except ProcessExecutionError as err:
+    except subp.ProcessExecutionError as err:
         LOG.warning("Unable to get lsb_release --all: %s", err)
         data = dict((v, "UNAVAILABLE") for v in fmap.values())
 
     return data
 
 
-def target_path(target, path=None):
-    # return 'path' inside target, accepting target as None
-    if target in (None, ""):
-        target = "/"
-    elif not isinstance(target, str):
-        raise ValueError("Unexpected input for target: %s" % target)
-    else:
-        target = os.path.abspath(target)
-        # abspath("//") returns "//" specifically for 2 slashes.
-        if target.startswith("//"):
-            target = target[1:]
-
-    if not path:
-        return target
-
-    # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /.
-    while len(path) and path[0] == "/":
-        path = path[1:]
-
-    return os.path.join(target, path)
-
-
 def decode_binary(blob, encoding='utf-8'):
     # Converts a binary type into a text type using given encoding.
     if isinstance(blob, str):
@@ -201,91 +181,6 @@ DMIDECODE_TO_DMI_SYS_MAPPING = {
 }
 
 
-class ProcessExecutionError(IOError):
-
-    MESSAGE_TMPL = ('%(description)s\n'
-                    'Command: %(cmd)s\n'
-                    'Exit code: %(exit_code)s\n'
-                    'Reason: %(reason)s\n'
-                    'Stdout: %(stdout)s\n'
-                    'Stderr: %(stderr)s')
-    empty_attr = '-'
-
-    def __init__(self, stdout=None, stderr=None,
-                 exit_code=None, cmd=None,
-                 description=None, reason=None,
-                 errno=None):
-        if not cmd:
-            self.cmd = self.empty_attr
-        else:
-            self.cmd = cmd
-
-        if not description:
-            if not exit_code and errno == ENOEXEC:
-                self.description = 'Exec format error. Missing #! in script?'
-            else:
-                self.description = 'Unexpected error while running command.'
-        else:
-            self.description = description
-
-        if not isinstance(exit_code, int):
-            self.exit_code = self.empty_attr
-        else:
-            self.exit_code = exit_code
-
-        if not stderr:
-            if stderr is None:
-                self.stderr = self.empty_attr
-            else:
-                self.stderr = stderr
-        else:
-            self.stderr = self._indent_text(stderr)
-
-        if not stdout:
-            if stdout is None:
-                self.stdout = self.empty_attr
-            else:
-                self.stdout = stdout
-        else:
-            self.stdout = self._indent_text(stdout)
-
-        if reason:
-            self.reason = reason
-        else:
-            self.reason = self.empty_attr
-
-        self.errno = errno
-        message = self.MESSAGE_TMPL % {
-            'description': self._ensure_string(self.description),
-            'cmd': self._ensure_string(self.cmd),
-            'exit_code': self._ensure_string(self.exit_code),
-            'stdout': self._ensure_string(self.stdout),
-            'stderr': self._ensure_string(self.stderr),
-            'reason': self._ensure_string(self.reason),
-        }
-        IOError.__init__(self, message)
-
-    def _ensure_string(self, text):
-        """
-        if data is bytes object, decode
-        """
-        return text.decode() if isinstance(text, bytes) else text
-
-    def _indent_text(self, text, indent_level=8):
-        """
-        indent text on all but the first line, allowing for easy to read output
-        """
-        cr = '\n'
-        indent = ' ' * indent_level
-        # if input is bytes, return bytes
-        if isinstance(text, bytes):
-            cr = cr.encode()
-            indent = indent.encode()
-        # remove any newlines at end of text first to prevent unneeded blank
-        # line in output
-        return text.rstrip(cr).replace(cr, cr + indent)
-
-
 class SeLinuxGuard(object):
     def __init__(self, path, recursive=False):
         # Late import since it might not always
@@ -875,8 +770,8 @@ def runparts(dirp, skip_no_exist=True, exe_prefix=None):
         if os.path.isfile(exe_path) and os.access(exe_path, os.X_OK):
             attempted.append(exe_path)
             try:
-                subp(prefix + [exe_path], capture=False)
-            except ProcessExecutionError as e:
+                subp.subp(prefix + [exe_path], capture=False)
+            except subp.ProcessExecutionError as e:
                 logexc(LOG, "Failed running %s [%s]", exe_path, e.exit_code)
                 failed.append(e)
 
@@ -1271,10 +1166,10 @@ def find_devs_with_netbsd(criteria=None, oformat='device',
             label = criteria.lstrip("LABEL=")
         if criteria.startswith("TYPE="):
             _type = criteria.lstrip("TYPE=")
-    out, _err = subp(['sysctl', '-n', 'hw.disknames'], rcs=[0])
+    out, _err = subp.subp(['sysctl', '-n', 'hw.disknames'], rcs=[0])
     for dev in out.split():
         if label or _type:
-            mscdlabel_out, _ = subp(['mscdlabel', dev], rcs=[0, 1])
+            mscdlabel_out, _ = subp.subp(['mscdlabel', dev], rcs=[0, 1])
         if label and not ('label "%s"' % label) in mscdlabel_out:
             continue
         if _type == "iso9660" and "ISO filesystem" not in mscdlabel_out:
@@ -1287,7 +1182,7 @@ def find_devs_with_netbsd(criteria=None, oformat='device',
 
 def find_devs_with_openbsd(criteria=None, oformat='device',
                            tag=None, no_cache=False, path=None):
-    out, _err = subp(['sysctl', '-n', 'hw.disknames'], rcs=[0])
+    out, _err = subp.subp(['sysctl', '-n', 'hw.disknames'], rcs=[0])
     devlist = []
     for entry in out.split(','):
         if not entry.endswith(':'):
@@ -1353,8 +1248,8 @@ def find_devs_with(criteria=None, oformat='device',
     cmd = blk_id_cmd + options
     # See man blkid for why 2 is added
     try:
-        (out, _err) = subp(cmd, rcs=[0, 2])
-    except ProcessExecutionError as e:
+        (out, _err) = subp.subp(cmd, rcs=[0, 2])
+    except subp.ProcessExecutionError as e:
         if e.errno == ENOENT:
             # blkid not found...
             out = ""
@@ -1389,7 +1284,7 @@ def blkid(devs=None, disable_cache=False):
     # we have to decode with 'replace' as shelx.split (called by
     # load_shell_content) can't take bytes.  So this is potentially
     # lossy of non-utf-8 chars in blkid output.
-    out, _ = subp(cmd, capture=True, decode="replace")
+    out, _ = subp.subp(cmd, capture=True, decode="replace")
     ret = {}
     for line in out.splitlines():
         dev, _, data = line.partition(":")
@@ -1709,7 +1604,7 @@ def unmounter(umount):
     finally:
         if umount:
             umount_cmd = ["umount", umount]
-            subp(umount_cmd)
+            subp.subp(umount_cmd)
 
 
 def mounts():
@@ -1720,7 +1615,7 @@ def mounts():
             mount_locs = load_file("/proc/mounts").splitlines()
             method = 'proc'
         else:
-            (mountoutput, _err) = subp("mount")
+            (mountoutput, _err) = subp.subp("mount")
             mount_locs = mountoutput.splitlines()
             method = 'mount'
         mountre = r'^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$'
@@ -1804,7 +1699,7 @@ def mount_cb(device, callback, data=None, mtype=None,
                         mountcmd.extend(['-t', mtype])
                     mountcmd.append(device)
                     mountcmd.append(tmpd)
-                    subp(mountcmd, update_env=update_env_for_mount)
+                    subp.subp(mountcmd, update_env=update_env_for_mount)
                     umount = tmpd  # This forces it to be unmounted (when set)
                     mountpoint = tmpd
                     break
@@ -1988,185 +1883,6 @@ def delete_dir_contents(dirname):
             del_file(node_fullpath)
 
 
-def subp_blob_in_tempfile(blob, *args, **kwargs):
-    """Write blob to a tempfile, and call subp with args, kwargs. Then cleanup.
-
-    'basename' as a kwarg allows providing the basename for the file.
-    The 'args' argument to subp will be updated with the full path to the
-    filename as the first argument.
-    """
-    basename = kwargs.pop('basename', "subp_blob")
-
-    if len(args) == 0 and 'args' not in kwargs:
-        args = [tuple()]
-
-    # Use tmpdir over tmpfile to avoid 'text file busy' on execute
-    with temp_utils.tempdir(needs_exe=True) as tmpd:
-        tmpf = os.path.join(tmpd, basename)
-        if 'args' in kwargs:
-            kwargs['args'] = [tmpf] + list(kwargs['args'])
-        else:
-            args = list(args)
-            args[0] = [tmpf] + args[0]
-
-        write_file(tmpf, blob, mode=0o700)
-        return subp(*args, **kwargs)
-
-
-def subp(args, data=None, rcs=None, env=None, capture=True,
-         combine_capture=False, shell=False,
-         logstring=False, decode="replace", target=None, update_env=None,
-         status_cb=None):
-    """Run a subprocess.
-
-    :param args: command to run in a list. [cmd, arg1, arg2...]
-    :param data: input to the command, made available on its stdin.
-    :param rcs:
-        a list of allowed return codes.  If subprocess exits with a value not
-        in this list, a ProcessExecutionError will be raised.  By default,
-        data is returned as a string.  See 'decode' parameter.
-    :param env: a dictionary for the command's environment.
-    :param capture:
-        boolean indicating if output should be captured.  If True, then stderr
-        and stdout will be returned.  If False, they will not be redirected.
-    :param combine_capture:
-        boolean indicating if stderr should be redirected to stdout. When True,
-        interleaved stderr and stdout will be returned as the first element of
-        a tuple, the second will be empty string or bytes (per decode).
-        if combine_capture is True, then output is captured independent of
-        the value of capture.
-    :param shell: boolean indicating if this should be run with a shell.
-    :param logstring:
-        the command will be logged to DEBUG.  If it contains info that should
-        not be logged, then logstring will be logged instead.
-    :param decode:
-        if False, no decoding will be done and returned stdout and stderr will
-        be bytes.  Other allowed values are 'strict', 'ignore', and 'replace'.
-        These values are passed through to bytes().decode() as the 'errors'
-        parameter.  There is no support for decoding to other than utf-8.
-    :param target:
-        not supported, kwarg present only to make function signature similar
-        to curtin's subp.
-    :param update_env:
-        update the enviornment for this command with this dictionary.
-        this will not affect the current processes os.environ.
-    :param status_cb:
-        call this fuction with a single string argument before starting
-        and after finishing.
-
-    :return
-        if not capturing, return is (None, None)
-        if capturing, stdout and stderr are returned.
-            if decode:
-                entries in tuple will be python2 unicode or python3 string
-            if not decode:
-                entries in tuple will be python2 string or python3 bytes
-    """
-
-    # not supported in cloud-init (yet), for now kept in the call signature
-    # to ease maintaining code shared between cloud-init and curtin
-    if target is not None:
-        raise ValueError("target arg not supported by cloud-init")
-
-    if rcs is None:
-        rcs = [0]
-
-    devnull_fp = None
-
-    if update_env:
-        if env is None:
-            env = os.environ
-        env = env.copy()
-        env.update(update_env)
-
-    if target_path(target) != "/":
-        args = ['chroot', target] + list(args)
-
-    if status_cb:
-        command = ' '.join(args) if isinstance(args, list) else args
-        status_cb('Begin run command: {command}\n'.format(command=command))
-    if not logstring:
-        LOG.debug(("Running command %s with allowed return codes %s"
-                   " (shell=%s, capture=%s)"),
-                  args, rcs, shell, 'combine' if combine_capture else capture)
-    else:
-        LOG.debug(("Running hidden command to protect sensitive "
-                   "input/output logstring: %s"), logstring)
-
-    stdin = None
-    stdout = None
-    stderr = None
-    if capture:
-        stdout = subprocess.PIPE
-        stderr = subprocess.PIPE
-    if combine_capture:
-        stdout = subprocess.PIPE
-        stderr = subprocess.STDOUT
-    if data is None:
-        # using devnull assures any reads get null, rather
-        # than possibly waiting on input.
-        devnull_fp = open(os.devnull)
-        stdin = devnull_fp
-    else:
-        stdin = subprocess.PIPE
-        if not isinstance(data, bytes):
-            data = data.encode()
-
-    # Popen converts entries in the arguments array from non-bytes to bytes.
-    # When locale is unset it may use ascii for that encoding which can
-    # cause UnicodeDecodeErrors. (LP: #1751051)
-    if isinstance(args, bytes):
-        bytes_args = args
-    elif isinstance(args, str):
-        bytes_args = args.encode("utf-8")
-    else:
-        bytes_args = [
-            x if isinstance(x, bytes) else x.encode("utf-8")
-            for x in args]
-    try:
-        sp = subprocess.Popen(bytes_args, stdout=stdout,
-                              stderr=stderr, stdin=stdin,
-                              env=env, shell=shell)
-        (out, err) = sp.communicate(data)
-    except OSError as e:
-        if status_cb:
-            status_cb('ERROR: End run command: invalid command provided\n')
-        raise ProcessExecutionError(
-            cmd=args, reason=e, errno=e.errno,
-            stdout="-" if decode else b"-",
-            stderr="-" if decode else b"-")
-    finally:
-        if devnull_fp:
-            devnull_fp.close()
-
-    # Just ensure blank instead of none.
-    if capture or combine_capture:
-        if not out:
-            out = b''
-        if not err:
-            err = b''
-    if decode:
-        def ldecode(data, m='utf-8'):
-            if not isinstance(data, bytes):
-                return data
-            return data.decode(m, decode)
-
-        out = ldecode(out)
-        err = ldecode(err)
-
-    rc = sp.returncode
-    if rc not in rcs:
-        if status_cb:
-            status_cb(
-                'ERROR: End run command: exit({code})\n'.format(code=rc))
-        raise ProcessExecutionError(stdout=out, stderr=err,
-                                    exit_code=rc,
-                                    cmd=args)
-    if status_cb:
-        status_cb('End run command: exit({code})\n'.format(code=rc))
-    return (out, err)
-
-
 def make_header(comment_char="#", base='created'):
     ci_ver = version.version_string()
     header = str(comment_char)
@@ -2232,7 +1948,7 @@ def is_container():
         try:
             # try to run a helper program. if it returns true/zero
             # then we're inside a container. otherwise, no
-            subp(helper)
+            subp.subp(helper)
             return True
         except (IOError, OSError):
             pass
@@ -2438,7 +2154,7 @@ def find_freebsd_part(fs):
         return splitted[2]
     elif splitted[2] in ['label', 'gpt', 'ufs']:
         target_label = fs[5:]
-        (part, _err) = subp(['glabel', 'status', '-s'])
+        (part, _err) = subp.subp(['glabel', 'status', '-s'])
         for labels in part.split("\n"):
             items = labels.split()
             if len(items) > 0 and items[0] == target_label:
@@ -2460,10 +2176,10 @@ def get_path_dev_freebsd(path, mnt_list):
 
 
 def get_mount_info_freebsd(path):
-    (result, err) = subp(['mount', '-p', path], rcs=[0, 1])
+    (result, err) = subp.subp(['mount', '-p', path], rcs=[0, 1])
     if len(err):
         # find a path if the input is not a mounting point
-        (mnt_list, err) = subp(['mount', '-p'])
+        (mnt_list, err) = subp.subp(['mount', '-p'])
         path_found = get_path_dev_freebsd(path, mnt_list)
         if (path_found is None):
             return None
@@ -2479,8 +2195,8 @@ def get_device_info_from_zpool(zpool):
         LOG.debug('Cannot get zpool info, no /dev/zfs')
         return None
     try:
-        (zpoolstatus, err) = subp(['zpool', 'status', zpool])
-    except ProcessExecutionError as err:
+        (zpoolstatus, err) = subp.subp(['zpool', 'status', zpool])
+    except subp.ProcessExecutionError as err:
         LOG.warning("Unable to get zpool status of %s: %s", zpool, err)
         return None
     if len(err):
@@ -2494,7 +2210,7 @@ def get_device_info_from_zpool(zpool):
 
 
 def parse_mount(path):
-    (mountoutput, _err) = subp(['mount'])
+    (mountoutput, _err) = subp.subp(['mount'])
     mount_locs = mountoutput.splitlines()
     # there are 2 types of mount outputs we have to parse therefore
     # the regex is a bit complex. to better understand this regex see:
@@ -2567,40 +2283,6 @@ def get_mount_info(path, log=LOG, get_mnt_opts=False):
         return parse_mount(path)
 
 
-def is_exe(fpath):
-    # return boolean indicating if fpath exists and is executable.
-    return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
-
-
-def which(program, search=None, target=None):
-    target = target_path(target)
-
-    if os.path.sep in program:
-        # if program had a '/' in it, then do not search PATH
-        # 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls
-        # so effectively we set cwd to / (or target)
-        if is_exe(target_path(target, program)):
-            return program
-
-    if search is None:
-        paths = [p.strip('"') for p in
-                 os.environ.get("PATH", "").split(os.pathsep)]
-        if target == "/":
-            search = paths
-        else:
-            search = [p for p in paths if p.startswith("/")]
-
-    # normalize path input
-    search = [os.path.abspath(p) for p in search]
-
-    for path in search:
-        ppath = os.path.sep.join((path, program))
-        if is_exe(target_path(target, ppath)):
-            return ppath
-
-    return None
-
-
 def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False):
     if args is None:
         args = []
@@ -2764,7 +2446,7 @@ def _call_dmidecode(key, dmidecode_path):
     """
     try:
         cmd = [dmidecode_path, "--string", key]
-        (result, _err) = subp(cmd)
+        (result, _err) = subp.subp(cmd)
         result = result.strip()
         LOG.debug("dmidecode returned '%s' for '%s'", result, key)
         if result.replace(".", "") == "":
@@ -2818,7 +2500,8 @@ def read_dmi_data(key):
         LOG.debug("dmidata is not supported on %s", uname_arch)
         return None
 
-    dmidecode_path = which('dmidecode')
+    print("hi, now its: %s\n", subp)
+    dmidecode_path = subp.which('dmidecode')
     if dmidecode_path:
         return _call_dmidecode(key, dmidecode_path)
 
@@ -2834,7 +2517,7 @@ def message_from_string(string):
 
 
 def get_installed_packages(target=None):
-    (out, _) = subp(['dpkg-query', '--list'], target=target, capture=True)
+    (out, _) = subp.subp(['dpkg-query', '--list'], target=target, capture=True)
 
     pkgs_inst = set()
     for line in out.splitlines():
@@ -2970,7 +2653,7 @@ def udevadm_settle(exists=None, timeout=None):
     if timeout:
         settle_cmd.extend(['--timeout=%s' % timeout])
 
-    return subp(settle_cmd)
+    return subp.subp(settle_cmd)
 
 
 def get_proc_ppid(pid):
diff --git a/packages/bddeb b/packages/bddeb
index 02ac2975..78b1c83b 100755
--- a/packages/bddeb
+++ b/packages/bddeb
@@ -24,6 +24,7 @@ def find_root():
 if "avoid-pep8-E402-import-not-top-of-file":
     # Use the util functions from cloudinit
     sys.path.insert(0, find_root())
+    from cloudinit import subp
     from cloudinit import util
     from cloudinit import temp_utils
     from cloudinit import templater
@@ -53,7 +54,7 @@ def run_helper(helper, args=None, strip=True):
     if args is None:
         args = []
     cmd = [util.abs_join(find_root(), 'tools', helper)] + args
-    (stdout, _stderr) = util.subp(cmd)
+    (stdout, _stderr) = subp.subp(cmd)
     if strip:
         stdout = stdout.strip()
     return stdout
@@ -67,7 +68,7 @@ def write_debian_folder(root, templ_data, cloud_util_deps):
 
     # Just copy debian/ dir and then update files
     pdeb_d = util.abs_join(find_root(), 'packages', 'debian')
-    util.subp(['cp', '-a', pdeb_d, deb_dir])
+    subp.subp(['cp', '-a', pdeb_d, deb_dir])
 
     # Fill in the change log template
     templater.render_to_file(util.abs_join(find_root(),
@@ -190,7 +191,7 @@ def main():
 
         print("Extracting temporary tarball %r" % (tarball))
         cmd = ['tar', '-xvzf', tarball_fp, '-C', tdir]
-        util.subp(cmd, capture=capture)
+        subp.subp(cmd, capture=capture)
 
         xdir = util.abs_join(tdir, "cloud-init-%s" % ver_data['version_long'])
         templ_data.update(ver_data)
@@ -203,7 +204,7 @@ def main():
             cmd = ['debuild', '--preserve-envvar', 'INIT_SYSTEM']
             if args.debuild_args:
                 cmd.extend(args.debuild_args)
-            util.subp(cmd, capture=capture)
+            subp.subp(cmd, capture=capture)
 
         link_fn = os.path.join(os.getcwd(), 'cloud-init_all.deb')
         link_dsc = os.path.join(os.getcwd(), 'cloud-init.dsc')
diff --git a/packages/brpm b/packages/brpm
index 1be8804c..56477d2e 100755
--- a/packages/brpm
+++ b/packages/brpm
@@ -24,6 +24,7 @@ def find_root():
 if "avoid-pep8-E402-import-not-top-of-file":
     # Use the util functions from cloudinit
     sys.path.insert(0, find_root())
+    from cloudinit import subp
     from cloudinit import templater
     from cloudinit import util
 
@@ -36,7 +37,7 @@ def run_helper(helper, args=None, strip=True):
     if args is None:
         args = []
     cmd = [util.abs_join(find_root(), 'tools', helper)] + args
-    (stdout, _stderr) = util.subp(cmd)
+    (stdout, _stderr) = subp.subp(cmd)
     if strip:
         stdout = stdout.strip()
     return stdout
@@ -174,7 +175,7 @@ def main():
         else:
             cmd = ['rpmbuild', '-ba', spec_fn]
 
-        util.subp(cmd, capture=capture)
+        subp.subp(cmd, capture=capture)
 
         # Copy the items built to our local dir
         globs = []
diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py
index f04d0cd4..6f74436d 100644
--- a/tests/cloud_tests/bddeb.py
+++ b/tests/cloud_tests/bddeb.py
@@ -6,7 +6,7 @@ from functools import partial
 import os
 import tempfile
 
-from cloudinit import util as c_util
+from cloudinit import subp
 from tests.cloud_tests import (config, LOG)
 from tests.cloud_tests import platforms
 from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single)
@@ -42,8 +42,8 @@ def build_deb(args, instance):
                'GIT_WORK_TREE': extract_dir}
 
     LOG.debug('creating tarball of cloud-init at: %s', local_tarball)
-    c_util.subp(['tar', 'cf', local_tarball, '--owner', 'root',
-                 '--group', 'root', '-C', args.cloud_init, '.'])
+    subp.subp(['tar', 'cf', local_tarball, '--owner', 'root',
+               '--group', 'root', '-C', args.cloud_init, '.'])
     LOG.debug('copying to remote system at: %s', remote_tarball)
     instance.push_file(local_tarball, remote_tarball)
 
diff --git a/tests/cloud_tests/platforms/lxd/image.py b/tests/cloud_tests/platforms/lxd/image.py
index b5de1f52..8934fb74 100644
--- a/tests/cloud_tests/platforms/lxd/image.py
+++ b/tests/cloud_tests/platforms/lxd/image.py
@@ -8,6 +8,7 @@ import tempfile
 
 from ..images import Image
 from .snapshot import LXDSnapshot
+from cloudinit import subp
 from cloudinit import util as c_util
 from tests.cloud_tests import util
 
@@ -81,8 +82,8 @@ class LXDImage(Image):
         @return_value: tuple of path to metadata tarball and rootfs tarball
         """
         # pylxd's image export feature doesn't do split exports, so use cmdline
-        c_util.subp(['lxc', 'image', 'export', self.pylxd_image.fingerprint,
-                     output_dir], capture=True)
+        subp.subp(['lxc', 'image', 'export', self.pylxd_image.fingerprint,
+                  output_dir], capture=True)
         tarballs = [p for p in os.listdir(output_dir) if p.endswith('tar.xz')]
         metadata = os.path.join(
             output_dir, next(p for p in tarballs if p.startswith('meta-')))
@@ -101,8 +102,8 @@ class LXDImage(Image):
         """
         alias = util.gen_instance_name(
             image_desc=str(self), use_desc='update-metadata')
-        c_util.subp(['lxc', 'image', 'import', metadata, rootfs,
-                     '--alias', alias], capture=True)
+        subp.subp(['lxc', 'image', 'import', metadata, rootfs,
+                   '--alias', alias], capture=True)
         self.pylxd_image = self.platform.query_image_by_alias(alias)
         return self.pylxd_image.fingerprint
 
diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py
index 2b804a62..b27b9848 100644
--- a/tests/cloud_tests/platforms/lxd/instance.py
+++ b/tests/cloud_tests/platforms/lxd/instance.py
@@ -7,7 +7,8 @@ import shutil
 import time
 from tempfile import mkdtemp
 
-from cloudinit.util import load_yaml, subp, ProcessExecutionError, which
+from cloudinit.subp import subp, ProcessExecutionError, which
+from cloudinit.util import load_yaml
 from tests.cloud_tests import LOG
 from tests.cloud_tests.util import PlatformError
 
diff --git a/tests/cloud_tests/platforms/nocloudkvm/image.py b/tests/cloud_tests/platforms/nocloudkvm/image.py
index bc2b6e75..ff5b6ad7 100644
--- a/tests/cloud_tests/platforms/nocloudkvm/image.py
+++ b/tests/cloud_tests/platforms/nocloudkvm/image.py
@@ -2,7 +2,7 @@
 
 """NoCloud KVM Image Base Class."""
 
-from cloudinit import util as c_util
+from cloudinit import subp
 
 import os
 import shutil
@@ -30,8 +30,8 @@ class NoCloudKVMImage(Image):
         self._img_path = os.path.join(self._workd,
                                       os.path.basename(self._orig_img_path))
 
-        c_util.subp(['qemu-img', 'create', '-f', 'qcow2',
-                    '-b', orig_img_path, self._img_path])
+        subp.subp(['qemu-img', 'create', '-f', 'qcow2',
+                   '-b', orig_img_path, self._img_path])
 
         super(NoCloudKVMImage, self).__init__(platform, config)
 
@@ -50,10 +50,10 @@ class NoCloudKVMImage(Image):
                       '--system-resolvconf', self._img_path,
                       '--', 'chroot', '_MOUNTPOINT_']
         try:
-            out, err = c_util.subp(mic_chroot + env_args + list(command),
-                                   data=stdin, decode=False)
+            out, err = subp.subp(mic_chroot + env_args + list(command),
+                                 data=stdin, decode=False)
             return (out, err, 0)
-        except c_util.ProcessExecutionError as e:
+        except subp.ProcessExecutionError as e:
             return (e.stdout, e.stderr, e.exit_code)
 
     def snapshot(self):
diff --git a/tests/cloud_tests/platforms/nocloudkvm/instance.py b/tests/cloud_tests/platforms/nocloudkvm/instance.py
index 96185b75..5140a11c 100644
--- a/tests/cloud_tests/platforms/nocloudkvm/instance.py
+++ b/tests/cloud_tests/platforms/nocloudkvm/instance.py
@@ -11,7 +11,7 @@ import uuid
 
 from ..instances import Instance
 from cloudinit.atomic_helper import write_json
-from cloudinit import util as c_util
+from cloudinit import subp
 from tests.cloud_tests import LOG, util
 
 # This domain contains reverse lookups for hostnames that are used.
@@ -110,8 +110,8 @@ class NoCloudKVMInstance(Instance):
         """Clean up instance."""
         if self.pid:
             try:
-                c_util.subp(['kill', '-9', self.pid])
-            except c_util.ProcessExecutionError:
+                subp.subp(['kill', '-9', self.pid])
+            except subp.ProcessExecutionError:
                 pass
 
         if self.pid_file:
@@ -143,8 +143,8 @@ class NoCloudKVMInstance(Instance):
 
         # meta-data can be yaml, but more easily pretty printed with json
         write_json(meta_data_file, self.meta_data)
-        c_util.subp(['cloud-localds', seed_file, user_data_file,
-                     meta_data_file])
+        subp.subp(['cloud-localds', seed_file, user_data_file,
+                   meta_data_file])
 
         return seed_file
 
diff --git a/tests/cloud_tests/platforms/nocloudkvm/platform.py b/tests/cloud_tests/platforms/nocloudkvm/platform.py
index 2d1480f5..53c8ebf2 100644
--- a/tests/cloud_tests/platforms/nocloudkvm/platform.py
+++ b/tests/cloud_tests/platforms/nocloudkvm/platform.py
@@ -12,6 +12,7 @@ from simplestreams import util as s_util
 from ..platforms import Platform
 from .image import NoCloudKVMImage
 from .instance import NoCloudKVMInstance
+from cloudinit import subp
 from cloudinit import util as c_util
 from tests.cloud_tests import util
 
@@ -84,8 +85,8 @@ class NoCloudKVMPlatform(Platform):
         """
         name = util.gen_instance_name(image_desc=image_desc, use_desc=use_desc)
         img_path = os.path.join(self.config['data_dir'], name + '.qcow2')
-        c_util.subp(['qemu-img', 'create', '-f', 'qcow2',
-                    '-b', src_img_path, img_path])
+        subp.subp(['qemu-img', 'create', '-f', 'qcow2',
+                   '-b', src_img_path, img_path])
 
         return NoCloudKVMInstance(self, name, img_path, properties, config,
                                   features, user_data, meta_data)
diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py
index bebdf1c6..58f65e52 100644
--- a/tests/cloud_tests/platforms/platforms.py
+++ b/tests/cloud_tests/platforms/platforms.py
@@ -7,6 +7,7 @@ import shutil
 from simplestreams import filters, mirrors
 from simplestreams import util as s_util
 
+from cloudinit import subp
 from cloudinit import util as c_util
 
 from tests.cloud_tests import util
@@ -48,10 +49,10 @@ class Platform(object):
         if os.path.exists(filename):
             c_util.del_file(filename)
 
-        c_util.subp(['ssh-keygen', '-m', 'PEM', '-t', 'rsa', '-b', '4096',
-                     '-f', filename, '-P', '',
-                     '-C', 'ubuntu@cloud_test'],
-                    capture=True)
+        subp.subp(['ssh-keygen', '-m', 'PEM', '-t', 'rsa', '-b', '4096',
+                   '-f', filename, '-P', '',
+                   '-C', 'ubuntu@cloud_test'],
+                  capture=True)
 
     @staticmethod
     def _query_streams(img_conf, img_filter):
diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py
index e65771b1..7dcccbdd 100644
--- a/tests/cloud_tests/util.py
+++ b/tests/cloud_tests/util.py
@@ -17,6 +17,7 @@ import time
 import yaml
 from contextlib import contextmanager
 
+from cloudinit import subp
 from cloudinit import util as c_util
 from tests.cloud_tests import LOG
 
@@ -232,8 +233,8 @@ def flat_tar(output, basedir, owner='root', group='root'):
     @param group: group archive files belong to
     @return_value: none
     """
-    c_util.subp(['tar', 'cf', output, '--owner', owner, '--group', group,
-                 '-C', basedir] + rel_files(basedir), capture=True)
+    subp.subp(['tar', 'cf', output, '--owner', owner, '--group', group,
+               '-C', basedir] + rel_files(basedir), capture=True)
 
 
 def parse_conf_list(entries, valid=None, boolean=False):
@@ -465,7 +466,7 @@ class TargetBase(object):
         return path
 
 
-class InTargetExecuteError(c_util.ProcessExecutionError):
+class InTargetExecuteError(subp.ProcessExecutionError):
     """Error type for in target commands that fail."""
 
     default_desc = 'Unexpected error while running command.'
diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py
index 9045e743..c5675249 100644
--- a/tests/unittests/test_builtin_handlers.py
+++ b/tests/unittests/test_builtin_handlers.py
@@ -15,6 +15,7 @@ from cloudinit.tests.helpers import (
 
 from cloudinit import handlers
 from cloudinit import helpers
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.handlers.cloud_config import CloudConfigPartHandler
@@ -66,7 +67,7 @@ class TestUpstartJobPartHandler(FilesystemMockingTestCase):
         util.ensure_dir("/etc/upstart")
 
         with mock.patch(self.mpath + 'SUITABLE_UPSTART', return_value=True):
-            with mock.patch.object(util, 'subp') as m_subp:
+            with mock.patch.object(subp, 'subp') as m_subp:
                 h = UpstartJobPartHandler(paths)
                 h.handle_part('', handlers.CONTENT_START,
                               None, None, None)
diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py
index 3119bfac..fc59d1d5 100644
--- a/tests/unittests/test_datasource/test_altcloud.py
+++ b/tests/unittests/test_datasource/test_altcloud.py
@@ -15,6 +15,7 @@ import shutil
 import tempfile
 
 from cloudinit import helpers
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.tests.helpers import CiTestCase, mock
@@ -286,7 +287,7 @@ class TestUserDataRhevm(CiTestCase):
     def test_modprobe_fails(self):
         '''Test user_data_rhevm() where modprobe fails.'''
 
-        self.m_modprobe_floppy.side_effect = util.ProcessExecutionError(
+        self.m_modprobe_floppy.side_effect = subp.ProcessExecutionError(
             "Failed modprobe")
         dsrc = dsac.DataSourceAltCloud({}, None, self.paths)
         self.assertEqual(False, dsrc.user_data_rhevm())
@@ -294,7 +295,7 @@ class TestUserDataRhevm(CiTestCase):
     def test_no_modprobe_cmd(self):
         '''Test user_data_rhevm() with no modprobe command.'''
 
-        self.m_modprobe_floppy.side_effect = util.ProcessExecutionError(
+        self.m_modprobe_floppy.side_effect = subp.ProcessExecutionError(
             "No such file or dir")
         dsrc = dsac.DataSourceAltCloud({}, None, self.paths)
         self.assertEqual(False, dsrc.user_data_rhevm())
@@ -302,7 +303,7 @@ class TestUserDataRhevm(CiTestCase):
     def test_udevadm_fails(self):
         '''Test user_data_rhevm() where udevadm fails.'''
 
-        self.m_udevadm_settle.side_effect = util.ProcessExecutionError(
+        self.m_udevadm_settle.side_effect = subp.ProcessExecutionError(
             "Failed settle.")
         dsrc = dsac.DataSourceAltCloud({}, None, self.paths)
         self.assertEqual(False, dsrc.user_data_rhevm())
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index b9f4d8fd..05552a1e 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -491,7 +491,7 @@ scbus-1 on xpt0 bus 0
             (dsaz, 'get_hostname', mock.MagicMock()),
             (dsaz, 'set_hostname', mock.MagicMock()),
             (dsaz, 'get_metadata_from_fabric', self.get_metadata_from_fabric),
-            (dsaz.util, 'which', lambda x: True),
+            (dsaz.subp, 'which', lambda x: True),
             (dsaz.util, 'read_dmi_data', mock.MagicMock(
                 side_effect=_dmi_mocks)),
             (dsaz.util, 'wait_for_files', mock.MagicMock(
@@ -1267,20 +1267,20 @@ scbus-1 on xpt0 bus 0
         expected_config['config'].append(blacklist_config)
         self.assertEqual(netconfig, expected_config)
 
-    @mock.patch(MOCKPATH + 'util.subp')
-    def test_get_hostname_with_no_args(self, subp):
+    @mock.patch(MOCKPATH + 'subp.subp')
+    def test_get_hostname_with_no_args(self, m_subp):
         dsaz.get_hostname()
-        subp.assert_called_once_with(("hostname",), capture=True)
+        m_subp.assert_called_once_with(("hostname",), capture=True)
 
-    @mock.patch(MOCKPATH + 'util.subp')
-    def test_get_hostname_with_string_arg(self, subp):
+    @mock.patch(MOCKPATH + 'subp.subp')
+    def test_get_hostname_with_string_arg(self, m_subp):
         dsaz.get_hostname(hostname_command="hostname")
-        subp.assert_called_once_with(("hostname",), capture=True)
+        m_subp.assert_called_once_with(("hostname",), capture=True)
 
-    @mock.patch(MOCKPATH + 'util.subp')
-    def test_get_hostname_with_iterable_arg(self, subp):
+    @mock.patch(MOCKPATH + 'subp.subp')
+    def test_get_hostname_with_iterable_arg(self, m_subp):
         dsaz.get_hostname(hostname_command=("hostname",))
-        subp.assert_called_once_with(("hostname",), capture=True)
+        m_subp.assert_called_once_with(("hostname",), capture=True)
 
 
 class TestAzureBounce(CiTestCase):
@@ -1302,7 +1302,7 @@ class TestAzureBounce(CiTestCase):
             mock.patch.object(dsaz, 'get_metadata_from_imds',
                               mock.MagicMock(return_value={})))
         self.patches.enter_context(
-            mock.patch.object(dsaz.util, 'which', lambda x: True))
+            mock.patch.object(dsaz.subp, 'which', lambda x: True))
         self.patches.enter_context(mock.patch.object(
             dsaz, '_get_random_seed', return_value='wild'))
 
@@ -1331,7 +1331,7 @@ class TestAzureBounce(CiTestCase):
         self.set_hostname = self.patches.enter_context(
             mock.patch.object(dsaz, 'set_hostname'))
         self.subp = self.patches.enter_context(
-            mock.patch(MOCKPATH + 'util.subp'))
+            mock.patch(MOCKPATH + 'subp.subp'))
         self.find_fallback_nic = self.patches.enter_context(
             mock.patch('cloudinit.net.find_fallback_nic', return_value='eth9'))
 
@@ -1414,7 +1414,7 @@ class TestAzureBounce(CiTestCase):
         cfg = {'hostname_bounce': {'policy': 'force'}}
         dsrc = self._get_ds(self.get_ovf_env_with_dscfg(host_name, cfg),
                             agent_command=['not', '__builtin__'])
-        patch_path = MOCKPATH + 'util.which'
+        patch_path = MOCKPATH + 'subp.which'
         with mock.patch(patch_path) as m_which:
             m_which.return_value = None
             ret = self._get_and_setup(dsrc)
@@ -1988,7 +1988,7 @@ class TestPreprovisioningPollIMDS(CiTestCase):
         self.assertEqual(report_ready_func.call_count, 0)
 
 
-@mock.patch(MOCKPATH + 'util.subp')
+@mock.patch(MOCKPATH + 'subp.subp')
 @mock.patch(MOCKPATH + 'util.write_file')
 @mock.patch(MOCKPATH + 'util.is_FreeBSD')
 @mock.patch('cloudinit.sources.helpers.netlink.'
@@ -2159,7 +2159,7 @@ class TestWBIsPlatformViable(CiTestCase):
             {'os.path.exists': False,
              # Non-matching Azure chassis-asset-tag
              'util.read_dmi_data': dsaz.AZURE_CHASSIS_ASSET_TAG + 'X',
-             'util.which': None},
+             'subp.which': None},
             dsaz._is_platform_viable, 'doesnotmatter'))
         self.assertIn(
             "DEBUG: Non-Azure DMI asset tag '{0}' discovered.\n".format(
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index 6344b0bb..71ef57f0 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -287,7 +287,7 @@ class TestOpenSSLManager(CiTestCase):
         self.addCleanup(patches.close)
 
         self.subp = patches.enter_context(
-            mock.patch.object(azure_helper.util, 'subp'))
+            mock.patch.object(azure_helper.subp, 'subp'))
         try:
             self.open = patches.enter_context(
                 mock.patch('__builtin__.open'))
diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py
index 83c2f753..e68168f2 100644
--- a/tests/unittests/test_datasource/test_cloudstack.py
+++ b/tests/unittests/test_datasource/test_cloudstack.py
@@ -41,7 +41,7 @@ class TestCloudStackPasswordFetching(CiTestCase):
     def _set_password_server_response(self, response_string):
         subp = mock.MagicMock(return_value=(response_string, ''))
         self.patches.enter_context(
-            mock.patch('cloudinit.sources.DataSourceCloudStack.util.subp',
+            mock.patch('cloudinit.sources.DataSourceCloudStack.subp.subp',
                        subp))
         return subp
 
diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py
index 486a2345..3ef7a4b2 100644
--- a/tests/unittests/test_datasource/test_ovf.py
+++ b/tests/unittests/test_datasource/test_ovf.py
@@ -10,6 +10,7 @@ import os
 from collections import OrderedDict
 from textwrap import dedent
 
+from cloudinit import subp
 from cloudinit import util
 from cloudinit.tests.helpers import CiTestCase, mock, wrap_and_call
 from cloudinit.helpers import Paths
@@ -401,8 +402,8 @@ class TestTransportIso9660(CiTestCase):
         self.assertTrue(dsovf.maybe_cdrom_device('xvdza1'))
 
 
-@mock.patch(MPATH + "util.which")
-@mock.patch(MPATH + "util.subp")
+@mock.patch(MPATH + "subp.which")
+@mock.patch(MPATH + "subp.subp")
 class TestTransportVmwareGuestinfo(CiTestCase):
     """Test the com.vmware.guestInfo transport implemented in
        transport_vmware_guestinfo."""
@@ -420,7 +421,7 @@ class TestTransportVmwareGuestinfo(CiTestCase):
     def test_notfound_on_exit_code_1(self, m_subp, m_which):
         """If vmware-rpctool exits 1, then must return not found."""
         m_which.return_value = self.rpctool_path
-        m_subp.side_effect = util.ProcessExecutionError(
+        m_subp.side_effect = subp.ProcessExecutionError(
             stdout="", stderr="No value found", exit_code=1, cmd=["unused"])
         self.assertEqual(NOT_FOUND, dsovf.transport_vmware_guestinfo())
         self.assertEqual(1, m_subp.call_count)
@@ -442,7 +443,7 @@ class TestTransportVmwareGuestinfo(CiTestCase):
     def test_notfound_and_warns_on_unexpected_exit_code(self, m_subp, m_which):
         """If vmware-rpctool exits non zero or 1, warnings should be logged."""
         m_which.return_value = self.rpctool_path
-        m_subp.side_effect = util.ProcessExecutionError(
+        m_subp.side_effect = subp.ProcessExecutionError(
             stdout=None, stderr="No value found", exit_code=2, cmd=["unused"])
         self.assertEqual(NOT_FOUND, dsovf.transport_vmware_guestinfo())
         self.assertEqual(1, m_subp.call_count)
diff --git a/tests/unittests/test_datasource/test_rbx.py b/tests/unittests/test_datasource/test_rbx.py
index 553af62e..d017510e 100644
--- a/tests/unittests/test_datasource/test_rbx.py
+++ b/tests/unittests/test_datasource/test_rbx.py
@@ -4,7 +4,7 @@ from cloudinit import helpers
 from cloudinit import distros
 from cloudinit.sources import DataSourceRbxCloud as ds
 from cloudinit.tests.helpers import mock, CiTestCase, populate_dir
-from cloudinit import util
+from cloudinit import subp
 
 DS_PATH = "cloudinit.sources.DataSourceRbxCloud"
 
@@ -157,7 +157,7 @@ class TestRbxDataSource(CiTestCase):
             expected
         )
 
-    @mock.patch(DS_PATH + '.util.subp')
+    @mock.patch(DS_PATH + '.subp.subp')
     def test_gratuitous_arp_run_standard_arping(self, m_subp):
         """Test handle run arping & parameters."""
         items = [
@@ -183,7 +183,7 @@ class TestRbxDataSource(CiTestCase):
         ], m_subp.call_args_list
         )
 
-    @mock.patch(DS_PATH + '.util.subp')
+    @mock.patch(DS_PATH + '.subp.subp')
     def test_handle_rhel_like_arping(self, m_subp):
         """Test handle on RHEL-like distros."""
         items = [
@@ -201,8 +201,8 @@ class TestRbxDataSource(CiTestCase):
         )
 
     @mock.patch(
-        DS_PATH + '.util.subp',
-        side_effect=util.ProcessExecutionError()
+        DS_PATH + '.subp.subp',
+        side_effect=subp.ProcessExecutionError()
     )
     def test_continue_on_arping_error(self, m_subp):
         """Continue when command error"""
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index f1ab1d7a..5847a384 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -32,8 +32,8 @@ from cloudinit.sources.DataSourceSmartOS import (
 from cloudinit.event import EventType
 
 from cloudinit import helpers as c_helpers
-from cloudinit.util import (
-    b64e, subp, ProcessExecutionError, which, write_file)
+from cloudinit.util import (b64e, write_file)
+from cloudinit.subp import (subp, ProcessExecutionError, which)
 
 from cloudinit.tests.helpers import (
     CiTestCase, mock, FilesystemMockingTestCase, skipIf)
@@ -667,7 +667,7 @@ class TestIdentifyFile(CiTestCase):
         with self.allow_subp(["file"]):
             self.assertEqual("text/plain", identify_file(fname))
 
-    @mock.patch(DSMOS + ".util.subp")
+    @mock.patch(DSMOS + ".subp.subp")
     def test_returns_none_on_error(self, m_subp):
         """On 'file' execution error, None should be returned."""
         m_subp.side_effect = ProcessExecutionError("FILE_FAILED", exit_code=99)
diff --git a/tests/unittests/test_distros/test_create_users.py b/tests/unittests/test_distros/test_create_users.py
index ef11784d..94ab052d 100644
--- a/tests/unittests/test_distros/test_create_users.py
+++ b/tests/unittests/test_distros/test_create_users.py
@@ -46,7 +46,7 @@ class MyBaseDistro(distros.Distro):
 
 
 @mock.patch("cloudinit.distros.util.system_is_snappy", return_value=False)
-@mock.patch("cloudinit.distros.util.subp")
+@mock.patch("cloudinit.distros.subp.subp")
 class TestCreateUser(CiTestCase):
 
     with_logs = True
@@ -240,7 +240,7 @@ class TestCreateUser(CiTestCase):
             [mock.call(set(['auth1']), user),  # not disabled
              mock.call(set(['key1']), 'foouser', options=disable_prefix)])
 
-    @mock.patch("cloudinit.distros.util.which")
+    @mock.patch("cloudinit.distros.subp.which")
     def test_lock_with_usermod_if_no_passwd(self, m_which, m_subp,
                                             m_is_snappy):
         """Lock uses usermod --lock if no 'passwd' cmd available."""
@@ -250,7 +250,7 @@ class TestCreateUser(CiTestCase):
             [mock.call(['usermod', '--lock', 'bob'])],
             m_subp.call_args_list)
 
-    @mock.patch("cloudinit.distros.util.which")
+    @mock.patch("cloudinit.distros.subp.which")
     def test_lock_with_passwd_if_available(self, m_which, m_subp,
                                            m_is_snappy):
         """Lock with only passwd will use passwd."""
@@ -260,7 +260,7 @@ class TestCreateUser(CiTestCase):
             [mock.call(['passwd', '-l', 'bob'])],
             m_subp.call_args_list)
 
-    @mock.patch("cloudinit.distros.util.which")
+    @mock.patch("cloudinit.distros.subp.which")
     def test_lock_raises_runtime_if_no_commands(self, m_which, m_subp,
                                                 m_is_snappy):
         """Lock with no commands available raises RuntimeError."""
diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py
index da16a797..7ff8240b 100644
--- a/tests/unittests/test_distros/test_debian.py
+++ b/tests/unittests/test_distros/test_debian.py
@@ -5,7 +5,7 @@ from cloudinit import util
 from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock)
 
 
-@mock.patch("cloudinit.distros.debian.util.subp")
+@mock.patch("cloudinit.distros.debian.subp.subp")
 class TestDebianApplyLocale(FilesystemMockingTestCase):
 
     def setUp(self):
diff --git a/tests/unittests/test_distros/test_freebsd.py b/tests/unittests/test_distros/test_freebsd.py
index 8af253a2..be565b04 100644
--- a/tests/unittests/test_distros/test_freebsd.py
+++ b/tests/unittests/test_distros/test_freebsd.py
@@ -8,7 +8,7 @@ import os
 
 class TestDeviceLookUp(CiTestCase):
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_find_freebsd_part_label(self, mock_subp):
         glabel_out = '''
 gptid/fa52d426-c337-11e6-8911-00155d4c5e47  N/A  da0p1
@@ -19,7 +19,7 @@ gptid/fa52d426-c337-11e6-8911-00155d4c5e47  N/A  da0p1
         res = find_freebsd_part("/dev/label/rootfs")
         self.assertEqual("da0p2", res)
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_find_freebsd_part_gpt(self, mock_subp):
         glabel_out = '''
                                 gpt/bootfs  N/A  vtbd0p1
diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py
index 02b334e3..6b6c1566 100644
--- a/tests/unittests/test_distros/test_generic.py
+++ b/tests/unittests/test_distros/test_generic.py
@@ -245,7 +245,7 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase):
         for d_name in ("ubuntu", "rhel"):
             cls = distros.fetch(d_name)
             d = cls(d_name, {}, None)
-            with mock.patch("cloudinit.util.subp") as m_subp:
+            with mock.patch("cloudinit.subp.subp") as m_subp:
                 d.expire_passwd("myuser")
             m_subp.assert_called_once_with(["passwd", "--expire", "myuser"])
 
@@ -253,7 +253,7 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase):
         """Test FreeBSD.expire_passwd uses the pw command."""
         cls = distros.fetch("freebsd")
         d = cls("freebsd", {}, None)
-        with mock.patch("cloudinit.util.subp") as m_subp:
+        with mock.patch("cloudinit.subp.subp") as m_subp:
             d.expire_passwd("myuser")
         m_subp.assert_called_once_with(
             ["pw", "usermod", "myuser", "-p", "01-Jan-1970"])
diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py
index 910207ca..8d7b09c8 100644
--- a/tests/unittests/test_distros/test_netconfig.py
+++ b/tests/unittests/test_distros/test_netconfig.py
@@ -12,6 +12,7 @@ from cloudinit import helpers
 from cloudinit import settings
 from cloudinit.tests.helpers import (
     FilesystemMockingTestCase, dir2dict)
+from cloudinit import subp
 from cloudinit import util
 
 
@@ -688,6 +689,6 @@ class TestNetCfgDistroArch(TestNetCfgDistroBase):
 
 
 def get_mode(path, target=None):
-    return os.stat(util.target_path(target, path)).st_mode & 0o777
+    return os.stat(subp.target_path(target, path)).st_mode & 0o777
 
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py
index a6faf0ef..fa48410a 100644
--- a/tests/unittests/test_distros/test_user_data_normalize.py
+++ b/tests/unittests/test_distros/test_user_data_normalize.py
@@ -307,7 +307,7 @@ class TestUGNormalize(TestCase):
         self.assertEqual({'default': False}, users['joe'])
         self.assertEqual({'default': False}, users['bob'])
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_create_snap_user(self, mock_subp):
         mock_subp.side_effect = [('{"username": "joe", "ssh-key-count": 1}\n',
                                   '')]
@@ -326,7 +326,7 @@ class TestUGNormalize(TestCase):
         mock_subp.assert_called_with(snapcmd, capture=True, logstring=snapcmd)
         self.assertEqual(username, 'joe')
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_create_snap_user_known(self, mock_subp):
         mock_subp.side_effect = [('{"username": "joe", "ssh-key-count": 1}\n',
                                   '')]
@@ -348,7 +348,7 @@ class TestUGNormalize(TestCase):
 
     @mock.patch('cloudinit.util.system_is_snappy')
     @mock.patch('cloudinit.util.is_group')
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_add_user_on_snappy_system(self, mock_subp, mock_isgrp,
                                        mock_snappy):
         mock_isgrp.return_value = False
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index 65a96090..fbb88e8a 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -6,6 +6,7 @@ import os
 from uuid import uuid4
 
 from cloudinit import safeyaml
+from cloudinit import subp
 from cloudinit import util
 from cloudinit.tests.helpers import (
     CiTestCase, dir2dict, populate_dir, populate_dir_with_ts)
@@ -160,8 +161,8 @@ class DsIdentifyBase(CiTestCase):
 
         rc = 0
         try:
-            out, err = util.subp(['sh', '-c', '. %s' % wrap], capture=True)
-        except util.ProcessExecutionError as e:
+            out, err = subp.subp(['sh', '-c', '. %s' % wrap], capture=True)
+        except subp.ProcessExecutionError as e:
             rc = e.exit_code
             out = e.stdout
             err = e.stderr
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
index 69009a44..e5382544 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
@@ -13,6 +13,7 @@ from cloudinit import cloud
 from cloudinit import distros
 from cloudinit import helpers
 from cloudinit import templater
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.config import cc_apt_configure
@@ -66,7 +67,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
     """
     def setUp(self):
         super(TestAptSourceConfigSourceList, self).setUp()
-        self.subp = util.subp
+        self.subp = subp.subp
         self.new_root = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, self.new_root)
 
@@ -176,7 +177,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
 
         # the second mock restores the original subp
         with mock.patch.object(util, 'write_file') as mockwrite:
-            with mock.patch.object(util, 'subp', self.subp):
+            with mock.patch.object(subp, 'subp', self.subp):
                 with mock.patch.object(Distro, 'get_primary_arch',
                                        return_value='amd64'):
                     cc_apt_configure.handle("notimportant", cfg, mycloud,
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py
index 0aa3d51a..b96fd4d4 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py
@@ -13,6 +13,7 @@ from unittest.mock import call
 from cloudinit import cloud
 from cloudinit import distros
 from cloudinit import helpers
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.config import cc_apt_configure
@@ -94,7 +95,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
     """TestAptSourceConfigSourceList - Class to test sources list rendering"""
     def setUp(self):
         super(TestAptSourceConfigSourceList, self).setUp()
-        self.subp = util.subp
+        self.subp = subp.subp
         self.new_root = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, self.new_root)
 
@@ -222,7 +223,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
 
         # the second mock restores the original subp
         with mock.patch.object(util, 'write_file') as mockwrite:
-            with mock.patch.object(util, 'subp', self.subp):
+            with mock.patch.object(subp, 'subp', self.subp):
                 with mock.patch.object(Distro, 'get_primary_arch',
                                        return_value='amd64'):
                     cc_apt_configure.handle("notimportant", cfg, mycloud,
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py
index 866752ef..f2349157 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py
@@ -14,6 +14,7 @@ from unittest.mock import call
 
 from cloudinit.config import cc_apt_configure
 from cloudinit import gpg
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.tests.helpers import TestCase
@@ -271,7 +272,7 @@ class TestAptSourceConfig(TestCase):
         """
         cfg = self.wrapv1conf(cfg)
 
-        with mock.patch.object(util, 'subp',
+        with mock.patch.object(subp, 'subp',
                                return_value=('fakekey 1234', '')) as mockobj:
             cc_apt_configure.handle("test", cfg, self.fakecloud, None, None)
 
@@ -356,7 +357,7 @@ class TestAptSourceConfig(TestCase):
         """
         cfg = self.wrapv1conf([cfg])
 
-        with mock.patch.object(util, 'subp') as mockobj:
+        with mock.patch.object(subp, 'subp') as mockobj:
             cc_apt_configure.handle("test", cfg, self.fakecloud, None, None)
 
         mockobj.assert_called_with(['apt-key', 'add', '-'],
@@ -398,7 +399,7 @@ class TestAptSourceConfig(TestCase):
                'filename': self.aptlistfile}
         cfg = self.wrapv1conf([cfg])
 
-        with mock.patch.object(util, 'subp') as mockobj:
+        with mock.patch.object(subp, 'subp') as mockobj:
             cc_apt_configure.handle("test", cfg, self.fakecloud, None, None)
 
         mockobj.assert_called_once_with(['apt-key', 'add', '-'],
@@ -413,7 +414,7 @@ class TestAptSourceConfig(TestCase):
                'filename': self.aptlistfile}
         cfg = self.wrapv1conf([cfg])
 
-        with mock.patch.object(util, 'subp',
+        with mock.patch.object(subp, 'subp',
                                return_value=('fakekey 1212', '')) as mockobj:
             cc_apt_configure.handle("test", cfg, self.fakecloud, None, None)
 
@@ -476,7 +477,7 @@ class TestAptSourceConfig(TestCase):
                'filename': self.aptlistfile}
         cfg = self.wrapv1conf([cfg])
 
-        with mock.patch.object(util, 'subp') as mockobj:
+        with mock.patch.object(subp, 'subp') as mockobj:
             cc_apt_configure.handle("test", cfg, self.fakecloud, None, None)
         mockobj.assert_called_once_with(['add-apt-repository',
                                          'ppa:smoser/cloud-init-test'],
@@ -495,7 +496,7 @@ class TestAptSourceConfig(TestCase):
                 'filename': self.aptlistfile3}
         cfg = self.wrapv1conf([cfg1, cfg2, cfg3])
 
-        with mock.patch.object(util, 'subp') as mockobj:
+        with mock.patch.object(subp, 'subp') as mockobj:
             cc_apt_configure.handle("test", cfg, self.fakecloud,
                                     None, None)
         calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test'],
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py
index aefe26c4..220100e2 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py
@@ -18,6 +18,7 @@ from cloudinit import cloud
 from cloudinit import distros
 from cloudinit import gpg
 from cloudinit import helpers
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.config import cc_apt_configure
@@ -221,7 +222,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
         """
         params = self._get_default_params()
 
-        with mock.patch("cloudinit.util.subp",
+        with mock.patch("cloudinit.subp.subp",
                         return_value=('fakekey 1234', '')) as mockobj:
             self._add_apt_sources(cfg, TARGET, template_params=params,
                                   aa_repo_match=self.matcher)
@@ -296,7 +297,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
                                              ' xenial main'),
                                   'key': "fakekey 4321"}}
 
-        with mock.patch.object(util, 'subp') as mockobj:
+        with mock.patch.object(subp, 'subp') as mockobj:
             self._add_apt_sources(cfg, TARGET, template_params=params,
                                   aa_repo_match=self.matcher)
 
@@ -318,7 +319,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
         params = self._get_default_params()
         cfg = {self.aptlistfile: {'key': "fakekey 4242"}}
 
-        with mock.patch.object(util, 'subp') as mockobj:
+        with mock.patch.object(subp, 'subp') as mockobj:
             self._add_apt_sources(cfg, TARGET, template_params=params,
                                   aa_repo_match=self.matcher)
 
@@ -333,7 +334,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
         params = self._get_default_params()
         cfg = {self.aptlistfile: {'keyid': "03683F77"}}
 
-        with mock.patch.object(util, 'subp',
+        with mock.patch.object(subp, 'subp',
                                return_value=('fakekey 1212', '')) as mockobj:
             self._add_apt_sources(cfg, TARGET, template_params=params,
                                   aa_repo_match=self.matcher)
@@ -416,7 +417,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
         params = self._get_default_params()
         cfg = {self.aptlistfile: {'source': 'ppa:smoser/cloud-init-test'}}
 
-        with mock.patch("cloudinit.util.subp") as mockobj:
+        with mock.patch("cloudinit.subp.subp") as mockobj:
             self._add_apt_sources(cfg, TARGET, template_params=params,
                                   aa_repo_match=self.matcher)
         mockobj.assert_any_call(['add-apt-repository',
@@ -432,7 +433,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
                self.aptlistfile2: {'source': 'ppa:smoser/cloud-init-test2'},
                self.aptlistfile3: {'source': 'ppa:smoser/cloud-init-test3'}}
 
-        with mock.patch("cloudinit.util.subp") as mockobj:
+        with mock.patch("cloudinit.subp.subp") as mockobj:
             self._add_apt_sources(cfg, TARGET, template_params=params,
                                   aa_repo_match=self.matcher)
         calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test'],
@@ -996,7 +997,7 @@ deb http://ubuntu.com/ubuntu/ xenial-proposed main""")
 
 class TestDebconfSelections(TestCase):
 
-    @mock.patch("cloudinit.config.cc_apt_configure.util.subp")
+    @mock.patch("cloudinit.config.cc_apt_configure.subp.subp")
     def test_set_sel_appends_newline_if_absent(self, m_subp):
         """Automatically append a newline to debconf-set-selections config."""
         selections = b'some/setting boolean true'
@@ -1081,7 +1082,7 @@ class TestDebconfSelections(TestCase):
         self.assertTrue(m_get_inst.called)
         self.assertEqual(m_dpkg_r.call_count, 0)
 
-    @mock.patch("cloudinit.config.cc_apt_configure.util.subp")
+    @mock.patch("cloudinit.config.cc_apt_configure.subp.subp")
     def test_dpkg_reconfigure_does_reconfigure(self, m_subp):
         target = "/foo-target"
 
@@ -1104,12 +1105,12 @@ class TestDebconfSelections(TestCase):
                     'cloud-init']
         self.assertEqual(expected, found)
 
-    @mock.patch("cloudinit.config.cc_apt_configure.util.subp")
+    @mock.patch("cloudinit.config.cc_apt_configure.subp.subp")
     def test_dpkg_reconfigure_not_done_on_no_data(self, m_subp):
         cc_apt_configure.dpkg_reconfigure([])
         m_subp.assert_not_called()
 
-    @mock.patch("cloudinit.config.cc_apt_configure.util.subp")
+    @mock.patch("cloudinit.config.cc_apt_configure.subp.subp")
     def test_dpkg_reconfigure_not_done_if_no_cleaners(self, m_subp):
         cc_apt_configure.dpkg_reconfigure(['pkgfoo', 'pkgbar'])
         m_subp.assert_not_called()
diff --git a/tests/unittests/test_handler/test_handler_bootcmd.py b/tests/unittests/test_handler/test_handler_bootcmd.py
index a76760fa..b53d60d4 100644
--- a/tests/unittests/test_handler/test_handler_bootcmd.py
+++ b/tests/unittests/test_handler/test_handler_bootcmd.py
@@ -2,7 +2,7 @@
 
 from cloudinit.config.cc_bootcmd import handle, schema
 from cloudinit.sources import DataSourceNone
-from cloudinit import (distros, helpers, cloud, util)
+from cloudinit import (distros, helpers, cloud, subp, util)
 from cloudinit.tests.helpers import (
     CiTestCase, mock, SchemaTestCaseMixin, skipUnlessJsonSchema)
 
@@ -36,7 +36,7 @@ class TestBootcmd(CiTestCase):
 
     def setUp(self):
         super(TestBootcmd, self).setUp()
-        self.subp = util.subp
+        self.subp = subp.subp
         self.new_root = self.tmp_dir()
 
     def _get_cloud(self, distro):
@@ -130,7 +130,7 @@ class TestBootcmd(CiTestCase):
 
         with mock.patch(self._etmpfile_path, FakeExtendedTempFile):
             with self.allow_subp(['/bin/sh']):
-                with self.assertRaises(util.ProcessExecutionError) as ctxt:
+                with self.assertRaises(subp.ProcessExecutionError) as ctxt:
                     handle('does-not-matter', valid_config, cc, LOG, [])
         self.assertIn(
             'Unexpected error while running command.\n'
diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py
index 286ef771..db0cdf9b 100644
--- a/tests/unittests/test_handler/test_handler_ca_certs.py
+++ b/tests/unittests/test_handler/test_handler_ca_certs.py
@@ -3,6 +3,7 @@
 from cloudinit import cloud
 from cloudinit.config import cc_ca_certs
 from cloudinit import helpers
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.tests.helpers import TestCase
@@ -228,7 +229,7 @@ class TestAddCaCerts(TestCase):
 
 class TestUpdateCaCerts(unittest.TestCase):
     def test_commands(self):
-        with mock.patch.object(util, 'subp') as mockobj:
+        with mock.patch.object(subp, 'subp') as mockobj:
             cc_ca_certs.update_ca_certs()
             mockobj.assert_called_once_with(
                 ["update-ca-certificates"], capture=False)
@@ -250,7 +251,7 @@ class TestRemoveDefaultCaCerts(TestCase):
                 mock.patch.object(util, 'delete_dir_contents'))
             mock_write = mocks.enter_context(
                 mock.patch.object(util, 'write_file'))
-            mock_subp = mocks.enter_context(mock.patch.object(util, 'subp'))
+            mock_subp = mocks.enter_context(mock.patch.object(subp, 'subp'))
 
             cc_ca_certs.remove_default_ca_certs()
 
diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py
index 8c476418..7918c609 100644
--- a/tests/unittests/test_handler/test_handler_chef.py
+++ b/tests/unittests/test_handler/test_handler_chef.py
@@ -41,7 +41,7 @@ class TestInstallChefOmnibus(HttprettyTestCase):
             httpretty.GET, cc_chef.OMNIBUS_URL, body=response, status=200)
         ret = (None, None)  # stdout, stderr but capture=False
 
-        with mock.patch("cloudinit.config.cc_chef.util.subp_blob_in_tempfile",
+        with mock.patch("cloudinit.config.cc_chef.subp_blob_in_tempfile",
                         return_value=ret) as m_subp_blob:
             cc_chef.install_chef_from_omnibus()
         # admittedly whitebox, but assuming subp_blob_in_tempfile works
@@ -52,7 +52,7 @@ class TestInstallChefOmnibus(HttprettyTestCase):
             m_subp_blob.call_args_list)
 
     @mock.patch('cloudinit.config.cc_chef.url_helper.readurl')
-    @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile')
+    @mock.patch('cloudinit.config.cc_chef.subp_blob_in_tempfile')
     def test_install_chef_from_omnibus_retries_url(self, m_subp_blob, m_rdurl):
         """install_chef_from_omnibus retries OMNIBUS_URL upon failure."""
 
@@ -81,7 +81,7 @@ class TestInstallChefOmnibus(HttprettyTestCase):
             m_subp_blob.call_args_list[0][1])
 
     @mock.patch("cloudinit.config.cc_chef.OMNIBUS_URL", OMNIBUS_URL_HTTP)
-    @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile')
+    @mock.patch('cloudinit.config.cc_chef.subp_blob_in_tempfile')
     def test_install_chef_from_omnibus_has_omnibus_version(self, m_subp_blob):
         """install_chef_from_omnibus provides version arg to OMNIBUS_URL."""
         chef_outfile = self.tmp_path('chef.out', self.new_root)
diff --git a/tests/unittests/test_handler/test_handler_disk_setup.py b/tests/unittests/test_handler/test_handler_disk_setup.py
index 0e51f17a..4f4a57fa 100644
--- a/tests/unittests/test_handler/test_handler_disk_setup.py
+++ b/tests/unittests/test_handler/test_handler_disk_setup.py
@@ -44,7 +44,7 @@ class TestGetMbrHddSize(TestCase):
         super(TestGetMbrHddSize, self).setUp()
         self.patches = ExitStack()
         self.subp = self.patches.enter_context(
-            mock.patch.object(cc_disk_setup.util, 'subp'))
+            mock.patch.object(cc_disk_setup.subp, 'subp'))
 
     def tearDown(self):
         super(TestGetMbrHddSize, self).tearDown()
@@ -173,7 +173,7 @@ class TestUpdateFsSetupDevices(TestCase):
 @mock.patch('cloudinit.config.cc_disk_setup.find_device_node',
             return_value=('/dev/xdb1', False))
 @mock.patch('cloudinit.config.cc_disk_setup.device_type', return_value=None)
-@mock.patch('cloudinit.config.cc_disk_setup.util.subp', return_value=('', ''))
+@mock.patch('cloudinit.config.cc_disk_setup.subp.subp', return_value=('', ''))
 class TestMkfsCommandHandling(CiTestCase):
 
     with_logs = True
@@ -204,7 +204,7 @@ class TestMkfsCommandHandling(CiTestCase):
         subp.assert_called_once_with(
             'mkfs -t ext4 -L with_cmd /dev/xdb1', shell=True)
 
-    @mock.patch('cloudinit.config.cc_disk_setup.util.which')
+    @mock.patch('cloudinit.config.cc_disk_setup.subp.which')
     def test_overwrite_and_extra_opts_without_cmd(self, m_which, subp, *args):
         """mkfs observes extra_opts and overwrite settings when cmd is not
         present."""
@@ -222,7 +222,7 @@ class TestMkfsCommandHandling(CiTestCase):
              '-L', 'without_cmd', '-F', 'are', 'added'],
             shell=False)
 
-    @mock.patch('cloudinit.config.cc_disk_setup.util.which')
+    @mock.patch('cloudinit.config.cc_disk_setup.subp.which')
     def test_mkswap(self, m_which, subp, *args):
         """mkfs observes extra_opts and overwrite settings when cmd is not
         present."""
diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py
index 501bcca5..7f039b79 100644
--- a/tests/unittests/test_handler/test_handler_growpart.py
+++ b/tests/unittests/test_handler/test_handler_growpart.py
@@ -2,7 +2,7 @@
 
 from cloudinit import cloud
 from cloudinit.config import cc_growpart
-from cloudinit import util
+from cloudinit import subp
 
 from cloudinit.tests.helpers import TestCase
 
@@ -95,7 +95,7 @@ class TestConfig(TestCase):
     @mock.patch.dict("os.environ", clear=True)
     def test_no_resizers_auto_is_fine(self):
         with mock.patch.object(
-                util, 'subp',
+                subp, 'subp',
                 return_value=(HELP_GROWPART_NO_RESIZE, "")) as mockobj:
 
             config = {'growpart': {'mode': 'auto'}}
@@ -109,7 +109,7 @@ class TestConfig(TestCase):
     @mock.patch.dict("os.environ", clear=True)
     def test_no_resizers_mode_growpart_is_exception(self):
         with mock.patch.object(
-                util, 'subp',
+                subp, 'subp',
                 return_value=(HELP_GROWPART_NO_RESIZE, "")) as mockobj:
             config = {'growpart': {'mode': "growpart"}}
             self.assertRaises(
@@ -122,7 +122,7 @@ class TestConfig(TestCase):
     @mock.patch.dict("os.environ", clear=True)
     def test_mode_auto_prefers_growpart(self):
         with mock.patch.object(
-                util, 'subp',
+                subp, 'subp',
                 return_value=(HELP_GROWPART_RESIZE, "")) as mockobj:
             ret = cc_growpart.resizer_factory(mode="auto")
             self.assertIsInstance(ret, cc_growpart.ResizeGrowPart)
@@ -133,7 +133,7 @@ class TestConfig(TestCase):
     @mock.patch.dict("os.environ", clear=True)
     def test_mode_auto_falls_back_to_gpart(self):
         with mock.patch.object(
-                util, 'subp',
+                subp, 'subp',
                 return_value=("", HELP_GPART)) as mockobj:
             ret = cc_growpart.resizer_factory(mode="auto")
             self.assertIsInstance(ret, cc_growpart.ResizeGpart)
diff --git a/tests/unittests/test_handler/test_handler_landscape.py b/tests/unittests/test_handler/test_handler_landscape.py
index db92a7e2..7d165687 100644
--- a/tests/unittests/test_handler/test_handler_landscape.py
+++ b/tests/unittests/test_handler/test_handler_landscape.py
@@ -49,8 +49,8 @@ class TestLandscape(FilesystemMockingTestCase):
             "'landscape' key existed in config, but not a dict",
             str(context_manager.exception))
 
-    @mock.patch('cloudinit.config.cc_landscape.util')
-    def test_handler_restarts_landscape_client(self, m_util):
+    @mock.patch('cloudinit.config.cc_landscape.subp')
+    def test_handler_restarts_landscape_client(self, m_subp):
         """handler restarts lansdscape-client after install."""
         mycloud = self._get_cloud('ubuntu')
         cfg = {'landscape': {'client': {}}}
@@ -60,7 +60,7 @@ class TestLandscape(FilesystemMockingTestCase):
             cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None)
         self.assertEqual(
             [mock.call(['service', 'landscape-client', 'restart'])],
-            m_util.subp.call_args_list)
+            m_subp.subp.call_args_list)
 
     def test_handler_installs_client_and_creates_config_file(self):
         """Write landscape client.conf and install landscape-client."""
diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py
index 407aa6c4..47e7d804 100644
--- a/tests/unittests/test_handler/test_handler_locale.py
+++ b/tests/unittests/test_handler/test_handler_locale.py
@@ -84,7 +84,7 @@ class TestLocale(t_help.FilesystemMockingTestCase):
         util.write_file(locale_conf, 'LANG="en_US.UTF-8"\n')
         cfg = {'locale': 'C.UTF-8'}
         cc = self._get_cloud('ubuntu')
-        with mock.patch('cloudinit.distros.debian.util.subp') as m_subp:
+        with mock.patch('cloudinit.distros.debian.subp.subp') as m_subp:
             with mock.patch('cloudinit.distros.debian.LOCALE_CONF_FN',
                             locale_conf):
                 cc_locale.handle('cc_locale', cfg, cc, LOG, [])
diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py
index 40b521e5..21011204 100644
--- a/tests/unittests/test_handler/test_handler_lxd.py
+++ b/tests/unittests/test_handler/test_handler_lxd.py
@@ -31,13 +31,13 @@ class TestLxd(t_help.CiTestCase):
         return cc
 
     @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default")
-    @mock.patch("cloudinit.config.cc_lxd.util")
-    def test_lxd_init(self, mock_util, m_maybe_clean):
+    @mock.patch("cloudinit.config.cc_lxd.subp")
+    def test_lxd_init(self, mock_subp, m_maybe_clean):
         cc = self._get_cloud('ubuntu')
-        mock_util.which.return_value = True
+        mock_subp.which.return_value = True
         m_maybe_clean.return_value = None
         cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, [])
-        self.assertTrue(mock_util.which.called)
+        self.assertTrue(mock_subp.which.called)
         # no bridge config, so maybe_cleanup should not be called.
         self.assertFalse(m_maybe_clean.called)
         self.assertEqual(
@@ -45,14 +45,14 @@ class TestLxd(t_help.CiTestCase):
              mock.call(
                  ['lxd', 'init', '--auto', '--network-address=0.0.0.0',
                   '--storage-backend=zfs', '--storage-pool=poolname'])],
-            mock_util.subp.call_args_list)
+            mock_subp.subp.call_args_list)
 
     @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default")
-    @mock.patch("cloudinit.config.cc_lxd.util")
-    def test_lxd_install(self, mock_util, m_maybe_clean):
+    @mock.patch("cloudinit.config.cc_lxd.subp")
+    def test_lxd_install(self, mock_subp, m_maybe_clean):
         cc = self._get_cloud('ubuntu')
         cc.distro = mock.MagicMock()
-        mock_util.which.return_value = None
+        mock_subp.which.return_value = None
         cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, [])
         self.assertNotIn('WARN', self.logs.getvalue())
         self.assertTrue(cc.distro.install_packages.called)
@@ -62,23 +62,23 @@ class TestLxd(t_help.CiTestCase):
         self.assertEqual(sorted(install_pkg), ['lxd', 'zfsutils-linux'])
 
     @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default")
-    @mock.patch("cloudinit.config.cc_lxd.util")
-    def test_no_init_does_nothing(self, mock_util, m_maybe_clean):
+    @mock.patch("cloudinit.config.cc_lxd.subp")
+    def test_no_init_does_nothing(self, mock_subp, m_maybe_clean):
         cc = self._get_cloud('ubuntu')
         cc.distro = mock.MagicMock()
         cc_lxd.handle('cc_lxd', {'lxd': {}}, cc, self.logger, [])
         self.assertFalse(cc.distro.install_packages.called)
-        self.assertFalse(mock_util.subp.called)
+        self.assertFalse(mock_subp.subp.called)
         self.assertFalse(m_maybe_clean.called)
 
     @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default")
-    @mock.patch("cloudinit.config.cc_lxd.util")
-    def test_no_lxd_does_nothing(self, mock_util, m_maybe_clean):
+    @mock.patch("cloudinit.config.cc_lxd.subp")
+    def test_no_lxd_does_nothing(self, mock_subp, m_maybe_clean):
         cc = self._get_cloud('ubuntu')
         cc.distro = mock.MagicMock()
         cc_lxd.handle('cc_lxd', {'package_update': True}, cc, self.logger, [])
         self.assertFalse(cc.distro.install_packages.called)
-        self.assertFalse(mock_util.subp.called)
+        self.assertFalse(mock_subp.subp.called)
         self.assertFalse(m_maybe_clean.called)
 
     def test_lxd_debconf_new_full(self):
diff --git a/tests/unittests/test_handler/test_handler_mcollective.py b/tests/unittests/test_handler/test_handler_mcollective.py
index c013a538..6891e15f 100644
--- a/tests/unittests/test_handler/test_handler_mcollective.py
+++ b/tests/unittests/test_handler/test_handler_mcollective.py
@@ -136,8 +136,9 @@ class TestHandler(t_help.TestCase):
         cc = cloud.Cloud(ds, paths, {}, d, None)
         return cc
 
+    @t_help.mock.patch("cloudinit.config.cc_mcollective.subp")
     @t_help.mock.patch("cloudinit.config.cc_mcollective.util")
-    def test_mcollective_install(self, mock_util):
+    def test_mcollective_install(self, mock_util, mock_subp):
         cc = self._get_cloud('ubuntu')
         cc.distro = t_help.mock.MagicMock()
         mock_util.load_file.return_value = b""
@@ -147,8 +148,8 @@ class TestHandler(t_help.TestCase):
         install_pkg = cc.distro.install_packages.call_args_list[0][0][0]
         self.assertEqual(install_pkg, ('mcollective',))
 
-        self.assertTrue(mock_util.subp.called)
-        self.assertEqual(mock_util.subp.call_args_list[0][0][0],
+        self.assertTrue(mock_subp.subp.called)
+        self.assertEqual(mock_subp.subp.call_args_list[0][0][0],
                          ['service', 'mcollective', 'restart'])
 
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py
index 35e72bd1..80c53c83 100644
--- a/tests/unittests/test_handler/test_handler_mounts.py
+++ b/tests/unittests/test_handler/test_handler_mounts.py
@@ -155,8 +155,8 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase):
                        'mock_is_block_device',
                        return_value=True)
 
-        self.add_patch('cloudinit.config.cc_mounts.util.subp',
-                       'm_util_subp')
+        self.add_patch('cloudinit.config.cc_mounts.subp.subp',
+                       'm_subp_subp')
 
         self.add_patch('cloudinit.config.cc_mounts.util.mounts',
                        'mock_util_mounts',
@@ -268,7 +268,7 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase):
             fstab_new_content = fd.read()
             self.assertEqual(fstab_expected_content, fstab_new_content)
         cc_mounts.handle(None, cc, self.mock_cloud, self.mock_log, [])
-        self.m_util_subp.assert_has_calls([
+        self.m_subp_subp.assert_has_calls([
             mock.call(['mount', '-a']),
             mock.call(['systemctl', 'daemon-reload'])])
 
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 463d892a..92a33ec1 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -83,50 +83,50 @@ class TestNtp(FilesystemMockingTestCase):
         ntpconfig['template_name'] = os.path.basename(confpath)
         return ntpconfig
 
-    @mock.patch("cloudinit.config.cc_ntp.util")
-    def test_ntp_install(self, mock_util):
+    @mock.patch("cloudinit.config.cc_ntp.subp")
+    def test_ntp_install(self, mock_subp):
         """ntp_install_client runs install_func when check_exe is absent."""
-        mock_util.which.return_value = None  # check_exe not found.
+        mock_subp.which.return_value = None  # check_exe not found.
         install_func = mock.MagicMock()
         cc_ntp.install_ntp_client(install_func,
                                   packages=['ntpx'], check_exe='ntpdx')
-        mock_util.which.assert_called_with('ntpdx')
+        mock_subp.which.assert_called_with('ntpdx')
         install_func.assert_called_once_with(['ntpx'])
 
-    @mock.patch("cloudinit.config.cc_ntp.util")
-    def test_ntp_install_not_needed(self, mock_util):
+    @mock.patch("cloudinit.config.cc_ntp.subp")
+    def test_ntp_install_not_needed(self, mock_subp):
         """ntp_install_client doesn't install when check_exe is found."""
         client = 'chrony'
-        mock_util.which.return_value = [client]  # check_exe found.
+        mock_subp.which.return_value = [client]  # check_exe found.
         install_func = mock.MagicMock()
         cc_ntp.install_ntp_client(install_func, packages=[client],
                                   check_exe=client)
         install_func.assert_not_called()
 
-    @mock.patch("cloudinit.config.cc_ntp.util")
-    def test_ntp_install_no_op_with_empty_pkg_list(self, mock_util):
+    @mock.patch("cloudinit.config.cc_ntp.subp")
+    def test_ntp_install_no_op_with_empty_pkg_list(self, mock_subp):
         """ntp_install_client runs install_func with empty list"""
-        mock_util.which.return_value = None  # check_exe not found
+        mock_subp.which.return_value = None  # check_exe not found
         install_func = mock.MagicMock()
         cc_ntp.install_ntp_client(install_func, packages=[],
                                   check_exe='timesyncd')
         install_func.assert_called_once_with([])
 
-    @mock.patch("cloudinit.config.cc_ntp.util")
-    def test_reload_ntp_defaults(self, mock_util):
+    @mock.patch("cloudinit.config.cc_ntp.subp")
+    def test_reload_ntp_defaults(self, mock_subp):
         """Test service is restarted/reloaded (defaults)"""
         service = 'ntp_service_name'
         cmd = ['service', service, 'restart']
         cc_ntp.reload_ntp(service)
-        mock_util.subp.assert_called_with(cmd, capture=True)
+        mock_subp.subp.assert_called_with(cmd, capture=True)
 
-    @mock.patch("cloudinit.config.cc_ntp.util")
-    def test_reload_ntp_systemd(self, mock_util):
+    @mock.patch("cloudinit.config.cc_ntp.subp")
+    def test_reload_ntp_systemd(self, mock_subp):
         """Test service is restarted/reloaded (systemd)"""
         service = 'ntp_service_name'
         cc_ntp.reload_ntp(service, systemd=True)
         cmd = ['systemctl', 'reload-or-restart', service]
-        mock_util.subp.assert_called_with(cmd, capture=True)
+        mock_subp.subp.assert_called_with(cmd, capture=True)
 
     def test_ntp_rename_ntp_conf(self):
         """When NTP_CONF exists, rename_ntp moves it."""
@@ -440,9 +440,10 @@ class TestNtp(FilesystemMockingTestCase):
             cc_ntp.handle('notimportant', cfg, mycloud, None, None)
             self.assertEqual(0, m_select.call_count)
 
+    @mock.patch("cloudinit.config.cc_ntp.subp")
     @mock.patch('cloudinit.config.cc_ntp.select_ntp_client')
     @mock.patch("cloudinit.distros.Distro.uses_systemd")
-    def test_ntp_the_whole_package(self, m_sysd, m_select):
+    def test_ntp_the_whole_package(self, m_sysd, m_select, m_subp):
         """Test enabled config renders template, and restarts service """
         cfg = {'ntp': {'enabled': True}}
         for distro in cc_ntp.distros:
@@ -458,12 +459,12 @@ class TestNtp(FilesystemMockingTestCase):
                 # allow use of util.mergemanydict
                 m_util.mergemanydict.side_effect = util.mergemanydict
                 # default client is present
-                m_util.which.return_value = True
+                m_subp.which.return_value = True
                 # use the config 'enabled' value
                 m_util.is_false.return_value = util.is_false(
                     cfg['ntp']['enabled'])
                 cc_ntp.handle('notimportant', cfg, mycloud, None, None)
-                m_util.subp.assert_called_with(
+                m_subp.subp.assert_called_with(
                     ['systemctl', 'reload-or-restart',
                      service_name], capture=True)
             self.assertEqual(
@@ -503,7 +504,7 @@ class TestNtp(FilesystemMockingTestCase):
         expected_client = mycloud.distro.preferred_ntp_clients[0]
         self.assertEqual('ntp', expected_client)
 
-    @mock.patch('cloudinit.config.cc_ntp.util.which')
+    @mock.patch('cloudinit.config.cc_ntp.subp.which')
     def test_snappy_system_picks_timesyncd(self, m_which):
         """Test snappy systems prefer installed clients"""
 
@@ -528,7 +529,7 @@ class TestNtp(FilesystemMockingTestCase):
         self.assertEqual(sorted(expected_cfg), sorted(cfg))
         self.assertEqual(sorted(expected_cfg), sorted(result))
 
-    @mock.patch('cloudinit.config.cc_ntp.util.which')
+    @mock.patch('cloudinit.config.cc_ntp.subp.which')
     def test_ntp_distro_searches_all_preferred_clients(self, m_which):
         """Test select_ntp_client search all distro perferred clients """
         # nothing is installed
@@ -546,7 +547,7 @@ class TestNtp(FilesystemMockingTestCase):
             m_which.assert_has_calls(expected_calls)
             self.assertEqual(sorted(expected_cfg), sorted(cfg))
 
-    @mock.patch('cloudinit.config.cc_ntp.util.which')
+    @mock.patch('cloudinit.config.cc_ntp.subp.which')
     def test_user_cfg_ntp_client_auto_uses_distro_clients(self, m_which):
         """Test user_cfg.ntp_client='auto' defaults to distro search"""
         # nothing is installed
@@ -566,7 +567,7 @@ class TestNtp(FilesystemMockingTestCase):
 
     @mock.patch('cloudinit.config.cc_ntp.write_ntp_config_template')
     @mock.patch('cloudinit.cloud.Cloud.get_template_filename')
-    @mock.patch('cloudinit.config.cc_ntp.util.which')
+    @mock.patch('cloudinit.config.cc_ntp.subp.which')
     def test_ntp_custom_client_overrides_installed_clients(self, m_which,
                                                            m_tmpfn, m_write):
         """Test user client is installed despite other clients present """
@@ -582,7 +583,7 @@ class TestNtp(FilesystemMockingTestCase):
             m_install.assert_called_with([client])
             m_which.assert_called_with(client)
 
-    @mock.patch('cloudinit.config.cc_ntp.util.which')
+    @mock.patch('cloudinit.config.cc_ntp.subp.which')
     def test_ntp_system_config_overrides_distro_builtin_clients(self, m_which):
         """Test distro system_config overrides builtin preferred ntp clients"""
         system_client = 'chrony'
@@ -597,7 +598,7 @@ class TestNtp(FilesystemMockingTestCase):
             self.assertEqual(sorted(expected_cfg), sorted(result))
             m_which.assert_has_calls([])
 
-    @mock.patch('cloudinit.config.cc_ntp.util.which')
+    @mock.patch('cloudinit.config.cc_ntp.subp.which')
     def test_ntp_user_config_overrides_system_cfg(self, m_which):
         """Test user-data overrides system_config ntp_client"""
         system_client = 'chrony'
diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py
index 2506d18a..62388ac6 100644
--- a/tests/unittests/test_handler/test_handler_puppet.py
+++ b/tests/unittests/test_handler/test_handler_puppet.py
@@ -12,11 +12,11 @@ import textwrap
 LOG = logging.getLogger(__name__)
 
 
-@mock.patch('cloudinit.config.cc_puppet.util')
+@mock.patch('cloudinit.config.cc_puppet.subp.subp')
 @mock.patch('cloudinit.config.cc_puppet.os')
 class TestAutostartPuppet(CiTestCase):
 
-    def test_wb_autostart_puppet_updates_puppet_default(self, m_os, m_util):
+    def test_wb_autostart_puppet_updates_puppet_default(self, m_os, m_subp):
         """Update /etc/default/puppet to autostart if it exists."""
 
         def _fake_exists(path):
@@ -27,9 +27,9 @@ class TestAutostartPuppet(CiTestCase):
         self.assertEqual(
             [mock.call(['sed', '-i', '-e', 's/^START=.*/START=yes/',
                         '/etc/default/puppet'], capture=False)],
-            m_util.subp.call_args_list)
+            m_subp.call_args_list)
 
-    def test_wb_autostart_pupppet_enables_puppet_systemctl(self, m_os, m_util):
+    def test_wb_autostart_pupppet_enables_puppet_systemctl(self, m_os, m_subp):
         """If systemctl is present, enable puppet via systemctl."""
 
         def _fake_exists(path):
@@ -39,9 +39,9 @@ class TestAutostartPuppet(CiTestCase):
         cc_puppet._autostart_puppet(LOG)
         expected_calls = [mock.call(
             ['/bin/systemctl', 'enable', 'puppet.service'], capture=False)]
-        self.assertEqual(expected_calls, m_util.subp.call_args_list)
+        self.assertEqual(expected_calls, m_subp.call_args_list)
 
-    def test_wb_autostart_pupppet_enables_puppet_chkconfig(self, m_os, m_util):
+    def test_wb_autostart_pupppet_enables_puppet_chkconfig(self, m_os, m_subp):
         """If chkconfig is present, enable puppet via checkcfg."""
 
         def _fake_exists(path):
@@ -51,7 +51,7 @@ class TestAutostartPuppet(CiTestCase):
         cc_puppet._autostart_puppet(LOG)
         expected_calls = [mock.call(
             ['/sbin/chkconfig', 'puppet', 'on'], capture=False)]
-        self.assertEqual(expected_calls, m_util.subp.call_args_list)
+        self.assertEqual(expected_calls, m_subp.call_args_list)
 
 
 @mock.patch('cloudinit.config.cc_puppet._autostart_puppet')
@@ -81,7 +81,7 @@ class TestPuppetHandle(CiTestCase):
             "no 'puppet' configuration found", self.logs.getvalue())
         self.assertEqual(0, m_auto.call_count)
 
-    @mock.patch('cloudinit.config.cc_puppet.util.subp')
+    @mock.patch('cloudinit.config.cc_puppet.subp.subp')
     def test_handler_puppet_config_starts_puppet_service(self, m_subp, m_auto):
         """Cloud-config 'puppet' configuration starts puppet."""
         mycloud = self._get_cloud('ubuntu')
@@ -92,7 +92,7 @@ class TestPuppetHandle(CiTestCase):
             [mock.call(['service', 'puppet', 'start'], capture=False)],
             m_subp.call_args_list)
 
-    @mock.patch('cloudinit.config.cc_puppet.util.subp')
+    @mock.patch('cloudinit.config.cc_puppet.subp.subp')
     def test_handler_empty_puppet_config_installs_puppet(self, m_subp, m_auto):
         """Cloud-config empty 'puppet' configuration installs latest puppet."""
         mycloud = self._get_cloud('ubuntu')
@@ -103,7 +103,7 @@ class TestPuppetHandle(CiTestCase):
             [mock.call(('puppet', None))],
             mycloud.distro.install_packages.call_args_list)
 
-    @mock.patch('cloudinit.config.cc_puppet.util.subp')
+    @mock.patch('cloudinit.config.cc_puppet.subp.subp')
     def test_handler_puppet_config_installs_puppet_on_true(self, m_subp, _):
         """Cloud-config with 'puppet' key installs when 'install' is True."""
         mycloud = self._get_cloud('ubuntu')
@@ -114,7 +114,7 @@ class TestPuppetHandle(CiTestCase):
             [mock.call(('puppet', None))],
             mycloud.distro.install_packages.call_args_list)
 
-    @mock.patch('cloudinit.config.cc_puppet.util.subp')
+    @mock.patch('cloudinit.config.cc_puppet.subp.subp')
     def test_handler_puppet_config_installs_puppet_version(self, m_subp, _):
         """Cloud-config 'puppet' configuration can specify a version."""
         mycloud = self._get_cloud('ubuntu')
@@ -125,7 +125,7 @@ class TestPuppetHandle(CiTestCase):
             [mock.call(('puppet', '3.8'))],
             mycloud.distro.install_packages.call_args_list)
 
-    @mock.patch('cloudinit.config.cc_puppet.util.subp')
+    @mock.patch('cloudinit.config.cc_puppet.subp.subp')
     def test_handler_puppet_config_updates_puppet_conf(self, m_subp, m_auto):
         """When 'conf' is provided update values in PUPPET_CONF_PATH."""
         mycloud = self._get_cloud('ubuntu')
@@ -141,7 +141,7 @@ class TestPuppetHandle(CiTestCase):
         expected = '[agent]\nserver = puppetmaster.example.org\nother = 3\n\n'
         self.assertEqual(expected, content)
 
-    @mock.patch('cloudinit.config.cc_puppet.util.subp')
+    @mock.patch('cloudinit.config.cc_puppet.subp.subp')
     def test_handler_puppet_writes_csr_attributes_file(self, m_subp, m_auto):
         """When csr_attributes is provided
             creates file in PUPPET_CSR_ATTRIBUTES_PATH."""
diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py
index 9ce334ac..73237d68 100644
--- a/tests/unittests/test_handler/test_handler_runcmd.py
+++ b/tests/unittests/test_handler/test_handler_runcmd.py
@@ -2,7 +2,7 @@
 
 from cloudinit.config.cc_runcmd import handle, schema
 from cloudinit.sources import DataSourceNone
-from cloudinit import (distros, helpers, cloud, util)
+from cloudinit import (distros, helpers, cloud, subp, util)
 from cloudinit.tests.helpers import (
     CiTestCase, FilesystemMockingTestCase, SchemaTestCaseMixin,
     skipUnlessJsonSchema)
@@ -20,7 +20,7 @@ class TestRuncmd(FilesystemMockingTestCase):
 
     def setUp(self):
         super(TestRuncmd, self).setUp()
-        self.subp = util.subp
+        self.subp = subp.subp
         self.new_root = self.tmp_dir()
 
     def _get_cloud(self, distro):
diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py
index abecc53b..85167f19 100644
--- a/tests/unittests/test_handler/test_handler_seed_random.py
+++ b/tests/unittests/test_handler/test_handler_seed_random.py
@@ -17,6 +17,7 @@ from io import BytesIO
 from cloudinit import cloud
 from cloudinit import distros
 from cloudinit import helpers
+from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.sources import DataSourceNone
@@ -35,8 +36,8 @@ class TestRandomSeed(t_help.TestCase):
         self.unapply = []
 
         # by default 'which' has nothing in its path
-        self.apply_patches([(util, 'which', self._which)])
-        self.apply_patches([(util, 'subp', self._subp)])
+        self.apply_patches([(subp, 'which', self._which)])
+        self.apply_patches([(subp, 'subp', self._subp)])
         self.subp_called = []
         self.whichdata = {}
 
diff --git a/tests/unittests/test_handler/test_handler_spacewalk.py b/tests/unittests/test_handler/test_handler_spacewalk.py
index 410e6f77..26f7648f 100644
--- a/tests/unittests/test_handler/test_handler_spacewalk.py
+++ b/tests/unittests/test_handler/test_handler_spacewalk.py
@@ -1,7 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit.config import cc_spacewalk
-from cloudinit import util
+from cloudinit import subp
 
 from cloudinit.tests import helpers
 
@@ -19,20 +19,20 @@ class TestSpacewalk(helpers.TestCase):
         }
     }
 
-    @mock.patch("cloudinit.config.cc_spacewalk.util.subp")
-    def test_not_is_registered(self, mock_util_subp):
-        mock_util_subp.side_effect = util.ProcessExecutionError(exit_code=1)
+    @mock.patch("cloudinit.config.cc_spacewalk.subp.subp")
+    def test_not_is_registered(self, mock_subp):
+        mock_subp.side_effect = subp.ProcessExecutionError(exit_code=1)
         self.assertFalse(cc_spacewalk.is_registered())
 
-    @mock.patch("cloudinit.config.cc_spacewalk.util.subp")
-    def test_is_registered(self, mock_util_subp):
-        mock_util_subp.side_effect = None
+    @mock.patch("cloudinit.config.cc_spacewalk.subp.subp")
+    def test_is_registered(self, mock_subp):
+        mock_subp.side_effect = None
         self.assertTrue(cc_spacewalk.is_registered())
 
-    @mock.patch("cloudinit.config.cc_spacewalk.util.subp")
-    def test_do_register(self, mock_util_subp):
+    @mock.patch("cloudinit.config.cc_spacewalk.subp.subp")
+    def test_do_register(self, mock_subp):
         cc_spacewalk.do_register(**self.space_cfg['spacewalk'])
-        mock_util_subp.assert_called_with([
+        mock_subp.assert_called_with([
             'rhnreg_ks',
             '--serverUrl', 'https://localhost/XMLRPC',
             '--profilename', 'test',
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index 23626395..a56022ef 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -8,6 +8,7 @@ from cloudinit.net import (
     renderers, sysconfig)
 from cloudinit.sources.helpers import openstack
 from cloudinit import temp_utils
+from cloudinit import subp
 from cloudinit import util
 from cloudinit import safeyaml as yaml
 
@@ -3192,7 +3193,7 @@ USERCTL=no
     def test_check_ifcfg_rh(self):
         """ifcfg-rh plugin is added NetworkManager.conf if conf present."""
         render_dir = self.tmp_dir()
-        nm_cfg = util.target_path(render_dir, path=self.nm_cfg_file)
+        nm_cfg = subp.target_path(render_dir, path=self.nm_cfg_file)
         util.ensure_dir(os.path.dirname(nm_cfg))
 
         # write a template nm.conf, note plugins is a list here
@@ -3215,7 +3216,7 @@ USERCTL=no
         """ifcfg-rh plugin is append when plugins is a string."""
         render_dir = self.tmp_path("render")
         os.makedirs(render_dir)
-        nm_cfg = util.target_path(render_dir, path=self.nm_cfg_file)
+        nm_cfg = subp.target_path(render_dir, path=self.nm_cfg_file)
         util.ensure_dir(os.path.dirname(nm_cfg))
 
         # write a template nm.conf, note plugins is a value here
@@ -3240,7 +3241,7 @@ USERCTL=no
         """enable_ifcfg_plugin creates plugins value if missing."""
         render_dir = self.tmp_path("render")
         os.makedirs(render_dir)
-        nm_cfg = util.target_path(render_dir, path=self.nm_cfg_file)
+        nm_cfg = subp.target_path(render_dir, path=self.nm_cfg_file)
         util.ensure_dir(os.path.dirname(nm_cfg))
 
         # write a template nm.conf, note plugins is missing
@@ -3920,7 +3921,7 @@ class TestNetplanCleanDefault(CiTestCase):
         files = sorted(populate_dir(tmpd, content))
         netplan._clean_default(target=tmpd)
         found = [t for t in files if os.path.exists(t)]
-        expected = [util.target_path(tmpd, f) for f in (astamp, anet, ayaml)]
+        expected = [subp.target_path(tmpd, f) for f in (astamp, anet, ayaml)]
         self.assertEqual(sorted(expected), found)
 
 
@@ -3933,7 +3934,7 @@ class TestNetplanPostcommands(CiTestCase):
 
     @mock.patch.object(netplan.Renderer, '_netplan_generate')
     @mock.patch.object(netplan.Renderer, '_net_setup_link')
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_netplan_render_calls_postcmds(self, mock_subp,
                                            mock_netplan_generate,
                                            mock_net_setup_link):
@@ -3947,7 +3948,7 @@ class TestNetplanPostcommands(CiTestCase):
         render_target = 'netplan.yaml'
         renderer = netplan.Renderer(
             {'netplan_path': render_target, 'postcmds': True})
-        mock_subp.side_effect = iter([util.ProcessExecutionError])
+        mock_subp.side_effect = iter([subp.ProcessExecutionError])
         renderer.render_network_state(ns, target=render_dir)
 
         mock_netplan_generate.assert_called_with(run=True)
@@ -3955,7 +3956,7 @@ class TestNetplanPostcommands(CiTestCase):
 
     @mock.patch('cloudinit.util.SeLinuxGuard')
     @mock.patch.object(netplan, "get_devicelist")
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_netplan_postcmds(self, mock_subp, mock_devlist, mock_sel):
         mock_sel.__enter__ = mock.Mock(return_value=False)
         mock_sel.__exit__ = mock.Mock()
@@ -3971,7 +3972,7 @@ class TestNetplanPostcommands(CiTestCase):
         renderer = netplan.Renderer(
             {'netplan_path': render_target, 'postcmds': True})
         mock_subp.side_effect = iter([
-            util.ProcessExecutionError,
+            subp.ProcessExecutionError,
             ('', ''),
             ('', ''),
         ])
@@ -4260,7 +4261,7 @@ class TestNetplanRoundTrip(CiTestCase):
 
     def setUp(self):
         super(TestNetplanRoundTrip, self).setUp()
-        self.add_patch('cloudinit.net.netplan.util.subp', 'm_subp')
+        self.add_patch('cloudinit.net.netplan.subp.subp', 'm_subp')
         self.m_subp.return_value = (self.NETPLAN_INFO_OUT, '')
 
     def _render_and_read(self, network_config=None, state=None,
@@ -5157,7 +5158,7 @@ def _gzip_data(data):
 
 class TestRenameInterfaces(CiTestCase):
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_rename_all(self, mock_subp):
         renames = [
             ('00:11:22:33:44:55', 'interface0', 'virtio_net', '0x3'),
@@ -5188,7 +5189,7 @@ class TestRenameInterfaces(CiTestCase):
                       capture=True),
         ])
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_rename_no_driver_no_device_id(self, mock_subp):
         renames = [
             ('00:11:22:33:44:55', 'interface0', None, None),
@@ -5219,7 +5220,7 @@ class TestRenameInterfaces(CiTestCase):
                       capture=True),
         ])
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_rename_all_bounce(self, mock_subp):
         renames = [
             ('00:11:22:33:44:55', 'interface0', 'virtio_net', '0x3'),
@@ -5254,7 +5255,7 @@ class TestRenameInterfaces(CiTestCase):
             mock.call(['ip', 'link', 'set', 'interface2', 'up'], capture=True)
         ])
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_rename_duplicate_macs(self, mock_subp):
         renames = [
             ('00:11:22:33:44:55', 'eth0', 'hv_netsvc', '0x3'),
@@ -5283,7 +5284,7 @@ class TestRenameInterfaces(CiTestCase):
                       capture=True),
         ])
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_rename_duplicate_macs_driver_no_devid(self, mock_subp):
         renames = [
             ('00:11:22:33:44:55', 'eth0', 'hv_netsvc', None),
@@ -5312,7 +5313,7 @@ class TestRenameInterfaces(CiTestCase):
                       capture=True),
         ])
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_rename_multi_mac_dups(self, mock_subp):
         renames = [
             ('00:11:22:33:44:55', 'eth0', 'hv_netsvc', '0x3'),
@@ -5351,7 +5352,7 @@ class TestRenameInterfaces(CiTestCase):
                       capture=True),
         ])
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_rename_macs_case_insensitive(self, mock_subp):
         """_rename_interfaces must support upper or lower case macs."""
         renames = [
diff --git a/tests/unittests/test_net_freebsd.py b/tests/unittests/test_net_freebsd.py
index 48296c30..414b4830 100644
--- a/tests/unittests/test_net_freebsd.py
+++ b/tests/unittests/test_net_freebsd.py
@@ -7,7 +7,7 @@ SAMPLE_FREEBSD_IFCONFIG_OUT = readResource("netinfo/freebsd-ifconfig-output")
 
 class TestInterfacesByMac(CiTestCase):
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     @mock.patch('cloudinit.util.is_FreeBSD')
     def test_get_interfaces_by_mac(self, mock_is_FreeBSD, mock_subp):
         mock_is_FreeBSD.return_value = True
diff --git a/tests/unittests/test_render_cloudcfg.py b/tests/unittests/test_render_cloudcfg.py
index 8b1e6042..393a78b1 100644
--- a/tests/unittests/test_render_cloudcfg.py
+++ b/tests/unittests/test_render_cloudcfg.py
@@ -5,6 +5,7 @@ import sys
 
 import pytest
 
+from cloudinit import subp
 from cloudinit import util
 
 # TODO(Look to align with tools.render-cloudcfg or cloudinit.distos.OSFAMILIES)
@@ -20,7 +21,7 @@ class TestRenderCloudCfg:
     @pytest.mark.parametrize('variant', (DISTRO_VARIANTS))
     def test_variant_sets_distro_in_cloud_cfg(self, variant, tmpdir):
         outfile = tmpdir.join('outcfg').strpath
-        util.subp(
+        subp.subp(
             self.cmd + ['--variant', variant, self.tmpl_path, outfile])
         with open(outfile) as stream:
             system_cfg = util.load_yaml(stream.read())
@@ -31,7 +32,7 @@ class TestRenderCloudCfg:
     @pytest.mark.parametrize('variant', (DISTRO_VARIANTS))
     def test_variant_sets_default_user_in_cloud_cfg(self, variant, tmpdir):
         outfile = tmpdir.join('outcfg').strpath
-        util.subp(
+        subp.subp(
             self.cmd + ['--variant', variant, self.tmpl_path, outfile])
         with open(outfile) as stream:
             system_cfg = util.load_yaml(stream.read())
@@ -49,7 +50,7 @@ class TestRenderCloudCfg:
         self, variant, renderers, tmpdir
     ):
         outfile = tmpdir.join('outcfg').strpath
-        util.subp(
+        subp.subp(
             self.cmd + ['--variant', variant, self.tmpl_path, outfile])
         with open(outfile) as stream:
             system_cfg = util.load_yaml(stream.read())
diff --git a/tests/unittests/test_reporting_hyperv.py b/tests/unittests/test_reporting_hyperv.py
index b3e083c6..fa8f8859 100644
--- a/tests/unittests/test_reporting_hyperv.py
+++ b/tests/unittests/test_reporting_hyperv.py
@@ -131,7 +131,7 @@ class TextKvpReporter(CiTestCase):
         self.assertEqual(0, len(kvps))
 
     @mock.patch('cloudinit.distros.uses_systemd')
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_get_boot_telemetry(self, m_subp, m_sysd):
         reporter = HyperVKvpReportingHandler(kvp_file_path=self.tmp_file_path)
         datetime_pattern = r"\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]"
diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py
index 4cd27eed..53d3cd5a 100644
--- a/tests/unittests/test_rh_subscription.py
+++ b/tests/unittests/test_rh_subscription.py
@@ -6,7 +6,7 @@ import copy
 import logging
 
 from cloudinit.config import cc_rh_subscription
-from cloudinit import util
+from cloudinit import subp
 
 from cloudinit.tests.helpers import CiTestCase, mock
 
@@ -56,7 +56,7 @@ class GoodTests(CiTestCase):
         '''
         reg = "The system has been registered with ID:" \
               " 12345678-abde-abcde-1234-1234567890abc"
-        m_sman_cli.side_effect = [util.ProcessExecutionError, (reg, 'bar')]
+        m_sman_cli.side_effect = [subp.ProcessExecutionError, (reg, 'bar')]
         self.handle(self.name, self.config, self.cloud_init,
                     self.log, self.args)
         self.assertIn(mock.call(['identity']), m_sman_cli.call_args_list)
@@ -93,7 +93,7 @@ class GoodTests(CiTestCase):
         reg = "The system has been registered with ID:" \
               " 12345678-abde-abcde-1234-1234567890abc"
         m_sman_cli.side_effect = [
-            util.ProcessExecutionError,
+            subp.ProcessExecutionError,
             (reg, 'bar'),
             ('Service level set to: self-support', ''),
             ('pool1\npool3\n', ''), ('pool2\n', ''), ('', ''),
@@ -161,7 +161,7 @@ class TestBadInput(CiTestCase):
 
     def test_no_password(self, m_sman_cli):
         '''Attempt to register without the password key/value.'''
-        m_sman_cli.side_effect = [util.ProcessExecutionError,
+        m_sman_cli.side_effect = [subp.ProcessExecutionError,
                                   (self.reg, 'bar')]
         self.handle(self.name, self.config_no_password, self.cloud_init,
                     self.log, self.args)
@@ -169,7 +169,7 @@ class TestBadInput(CiTestCase):
 
     def test_no_org(self, m_sman_cli):
         '''Attempt to register without the org key/value.'''
-        m_sman_cli.side_effect = [util.ProcessExecutionError]
+        m_sman_cli.side_effect = [subp.ProcessExecutionError]
         self.handle(self.name, self.config_no_key, self.cloud_init,
                     self.log, self.args)
         m_sman_cli.assert_called_with(['identity'])
@@ -182,7 +182,7 @@ class TestBadInput(CiTestCase):
 
     def test_service_level_without_auto(self, m_sman_cli):
         '''Attempt to register using service-level without auto-attach key.'''
-        m_sman_cli.side_effect = [util.ProcessExecutionError,
+        m_sman_cli.side_effect = [subp.ProcessExecutionError,
                                   (self.reg, 'bar')]
         self.handle(self.name, self.config_service, self.cloud_init,
                     self.log, self.args)
@@ -195,7 +195,7 @@ class TestBadInput(CiTestCase):
         '''
         Register with pools that are not in the format of a list
         '''
-        m_sman_cli.side_effect = [util.ProcessExecutionError,
+        m_sman_cli.side_effect = [subp.ProcessExecutionError,
                                   (self.reg, 'bar')]
         self.handle(self.name, self.config_badpool, self.cloud_init,
                     self.log, self.args)
@@ -208,7 +208,7 @@ class TestBadInput(CiTestCase):
         '''
         Register with repos that are not in the format of a list
         '''
-        m_sman_cli.side_effect = [util.ProcessExecutionError,
+        m_sman_cli.side_effect = [subp.ProcessExecutionError,
                                   (self.reg, 'bar')]
         self.handle(self.name, self.config_badrepo, self.cloud_init,
                     self.log, self.args)
@@ -222,7 +222,7 @@ class TestBadInput(CiTestCase):
         '''
         Attempt to register with a key that we don't know
         '''
-        m_sman_cli.side_effect = [util.ProcessExecutionError,
+        m_sman_cli.side_effect = [subp.ProcessExecutionError,
                                   (self.reg, 'bar')]
         self.handle(self.name, self.config_badkey, self.cloud_init,
                     self.log, self.args)
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 4b439267..737dda8b 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -1,25 +1,20 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import io
-import json
 import logging
 import os
 import re
 import shutil
 import stat
-import sys
 import tempfile
 import yaml
 from unittest import mock
 
+from cloudinit import subp
 from cloudinit import importer, util
 from cloudinit.tests import helpers
 
 
-BASH = util.which('bash')
-BOGUS_COMMAND = 'this-is-not-expected-to-be-a-program-name'
-
-
 class FakeSelinux(object):
 
     def __init__(self, match_what):
@@ -385,7 +380,7 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase):
         self.assertEqual(expected, util.parse_mount_info('/run/lock', lines))
 
     @mock.patch('cloudinit.util.os')
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_get_device_info_from_zpool(self, zpool_output, m_os):
         # mock /dev/zfs exists
         m_os.path.exists.return_value = True
@@ -408,17 +403,17 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase):
         self.assertIsNone(ret)
 
     @mock.patch('cloudinit.util.os')
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_get_device_info_from_zpool_handles_no_zpool(self, m_sub, m_os):
         """Handle case where there is no zpool command"""
         # mock /dev/zfs exists
         m_os.path.exists.return_value = True
-        m_sub.side_effect = util.ProcessExecutionError("No zpool cmd")
+        m_sub.side_effect = subp.ProcessExecutionError("No zpool cmd")
         ret = util.get_device_info_from_zpool('vmzroot')
         self.assertIsNone(ret)
 
     @mock.patch('cloudinit.util.os')
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_get_device_info_from_zpool_on_error(self, zpool_output, m_os):
         # mock /dev/zfs exists
         m_os.path.exists.return_value = True
@@ -430,7 +425,7 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase):
         ret = util.get_device_info_from_zpool('vmzroot')
         self.assertIsNone(ret)
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_parse_mount_with_ext(self, mount_out):
         mount_out.return_value = (
             helpers.readResource('mount_parse_ext.txt'), '')
@@ -447,7 +442,7 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase):
         ret = util.parse_mount('/not/existing/mount')
         self.assertIsNone(ret)
 
-    @mock.patch('cloudinit.util.subp')
+    @mock.patch('cloudinit.subp.subp')
     def test_parse_mount_with_zfs(self, mount_out):
         mount_out.return_value = (
             helpers.readResource('mount_parse_zfs.txt'), '')
@@ -513,13 +508,13 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase):
         """
         def _dmidecode_subp(cmd):
             if cmd[-1] != key:
-                raise util.ProcessExecutionError()
+                raise subp.ProcessExecutionError()
             return (content, error)
 
         self.patched_funcs.enter_context(
-            mock.patch.object(util, 'which', lambda _: True))
+            mock.patch("cloudinit.subp.which", side_effect=lambda _: True))
         self.patched_funcs.enter_context(
-            mock.patch.object(util, 'subp', _dmidecode_subp))
+            mock.patch("cloudinit.subp.subp", side_effect=_dmidecode_subp))
 
     def patch_mapping(self, new_mapping):
         self.patched_funcs.enter_context(
@@ -546,10 +541,12 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase):
 
     def test_dmidecode_not_used_on_arm(self):
         self.patch_mapping({})
+        print("current =%s", subp)
         self._create_sysfs_parent_directory()
         dmi_val = 'from-dmidecode'
         dmi_name = 'use-dmidecode'
         self._configure_dmidecode_return(dmi_name, dmi_val)
+        print("now =%s", subp)
 
         expected = {'armel': None, 'aarch64': dmi_val, 'x86_64': dmi_val}
         found = {}
@@ -560,6 +557,7 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase):
             for arch in expected:
                 m_uname.return_value = ('x-sysname', 'x-nodename',
                                         'x-release', 'x-version', arch)
+                print("now2 =%s", subp)
                 found[arch] = util.read_dmi_data(dmi_name)
         self.assertEqual(expected, found)
 
@@ -570,7 +568,7 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase):
 
     def test_none_returned_if_dmidecode_not_in_path(self):
         self.patched_funcs.enter_context(
-            mock.patch.object(util, 'which', lambda _: False))
+            mock.patch.object(subp, 'which', lambda _: False))
         self.patch_mapping({})
         self.assertIsNone(util.read_dmi_data('expect-fail'))
 
@@ -734,218 +732,6 @@ class TestReadSeeded(helpers.TestCase):
         self.assertEqual(found_ud, ud)
 
 
-class TestSubp(helpers.CiTestCase):
-    allowed_subp = [BASH, 'cat', helpers.CiTestCase.SUBP_SHELL_TRUE,
-                    BOGUS_COMMAND, sys.executable]
-
-    stdin2err = [BASH, '-c', 'cat >&2']
-    stdin2out = ['cat']
-    utf8_invalid = b'ab\xaadef'
-    utf8_valid = b'start \xc3\xa9 end'
-    utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7'
-    printenv = [BASH, '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--']
-
-    def printf_cmd(self, *args):
-        # bash's printf supports \xaa.  So does /usr/bin/printf
-        # but by using bash, we remove dependency on another program.
-        return([BASH, '-c', 'printf "$@"', 'printf'] + list(args))
-
-    def test_subp_handles_bytestrings(self):
-        """subp can run a bytestring command if shell is True."""
-        tmp_file = self.tmp_path('test.out')
-        cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file)
-        (out, _err) = util.subp(cmd.encode('utf-8'), shell=True)
-        self.assertEqual(u'', out)
-        self.assertEqual(u'', _err)
-        self.assertEqual('HI MOM\n', util.load_file(tmp_file))
-
-    def test_subp_handles_strings(self):
-        """subp can run a string command if shell is True."""
-        tmp_file = self.tmp_path('test.out')
-        cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file)
-        (out, _err) = util.subp(cmd, shell=True)
-        self.assertEqual(u'', out)
-        self.assertEqual(u'', _err)
-        self.assertEqual('HI MOM\n', util.load_file(tmp_file))
-
-    def test_subp_handles_utf8(self):
-        # The given bytes contain utf-8 accented characters as seen in e.g.
-        # the "deja dup" package in Ubuntu.
-        cmd = self.printf_cmd(self.utf8_valid_2)
-        (out, _err) = util.subp(cmd, capture=True)
-        self.assertEqual(out, self.utf8_valid_2.decode('utf-8'))
-
-    def test_subp_respects_decode_false(self):
-        (out, err) = util.subp(self.stdin2out, capture=True, decode=False,
-                               data=self.utf8_valid)
-        self.assertTrue(isinstance(out, bytes))
-        self.assertTrue(isinstance(err, bytes))
-        self.assertEqual(out, self.utf8_valid)
-
-    def test_subp_decode_ignore(self):
-        # this executes a string that writes invalid utf-8 to stdout
-        (out, _err) = util.subp(self.printf_cmd('abc\\xaadef'),
-                                capture=True, decode='ignore')
-        self.assertEqual(out, 'abcdef')
-
-    def test_subp_decode_strict_valid_utf8(self):
-        (out, _err) = util.subp(self.stdin2out, capture=True,
-                                decode='strict', data=self.utf8_valid)
-        self.assertEqual(out, self.utf8_valid.decode('utf-8'))
-
-    def test_subp_decode_invalid_utf8_replaces(self):
-        (out, _err) = util.subp(self.stdin2out, capture=True,
-                                data=self.utf8_invalid)
-        expected = self.utf8_invalid.decode('utf-8', 'replace')
-        self.assertEqual(out, expected)
-
-    def test_subp_decode_strict_raises(self):
-        args = []
-        kwargs = {'args': self.stdin2out, 'capture': True,
-                  'decode': 'strict', 'data': self.utf8_invalid}
-        self.assertRaises(UnicodeDecodeError, util.subp, *args, **kwargs)
-
-    def test_subp_capture_stderr(self):
-        data = b'hello world'
-        (out, err) = util.subp(self.stdin2err, capture=True,
-                               decode=False, data=data,
-                               update_env={'LC_ALL': 'C'})
-        self.assertEqual(err, data)
-        self.assertEqual(out, b'')
-
-    def test_subp_reads_env(self):
-        with mock.patch.dict("os.environ", values={'FOO': 'BAR'}):
-            out, _err = util.subp(self.printenv + ['FOO'], capture=True)
-        self.assertEqual('FOO=BAR', out.splitlines()[0])
-
-    def test_subp_env_and_update_env(self):
-        out, _err = util.subp(
-            self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True,
-            env={'FOO': 'BAR'},
-            update_env={'HOME': '/myhome', 'K2': 'V2'})
-        self.assertEqual(
-            ['FOO=BAR', 'HOME=/myhome', 'K1=', 'K2=V2'], out.splitlines())
-
-    def test_subp_update_env(self):
-        extra = {'FOO': 'BAR', 'HOME': '/root', 'K1': 'V1'}
-        with mock.patch.dict("os.environ", values=extra):
-            out, _err = util.subp(
-                self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True,
-                update_env={'HOME': '/myhome', 'K2': 'V2'})
-
-        self.assertEqual(
-            ['FOO=BAR', 'HOME=/myhome', 'K1=V1', 'K2=V2'], out.splitlines())
-
-    def test_subp_warn_missing_shebang(self):
-        """Warn on no #! in script"""
-        noshebang = self.tmp_path('noshebang')
-        util.write_file(noshebang, 'true\n')
-
-        os.chmod(noshebang, os.stat(noshebang).st_mode | stat.S_IEXEC)
-        with self.allow_subp([noshebang]):
-            self.assertRaisesRegex(util.ProcessExecutionError,
-                                   r'Missing #! in script\?',
-                                   util.subp, (noshebang,))
-
-    def test_subp_combined_stderr_stdout(self):
-        """Providing combine_capture as True redirects stderr to stdout."""
-        data = b'hello world'
-        (out, err) = util.subp(self.stdin2err, capture=True,
-                               combine_capture=True, decode=False, data=data)
-        self.assertEqual(b'', err)
-        self.assertEqual(data, out)
-
-    def test_returns_none_if_no_capture(self):
-        (out, err) = util.subp(self.stdin2out, data=b'', capture=False)
-        self.assertIsNone(err)
-        self.assertIsNone(out)
-
-    def test_exception_has_out_err_are_bytes_if_decode_false(self):
-        """Raised exc should have stderr, stdout as bytes if no decode."""
-        with self.assertRaises(util.ProcessExecutionError) as cm:
-            util.subp([BOGUS_COMMAND], decode=False)
-        self.assertTrue(isinstance(cm.exception.stdout, bytes))
-        self.assertTrue(isinstance(cm.exception.stderr, bytes))
-
-    def test_exception_has_out_err_are_bytes_if_decode_true(self):
-        """Raised exc should have stderr, stdout as string if no decode."""
-        with self.assertRaises(util.ProcessExecutionError) as cm:
-            util.subp([BOGUS_COMMAND], decode=True)
-        self.assertTrue(isinstance(cm.exception.stdout, str))
-        self.assertTrue(isinstance(cm.exception.stderr, str))
-
-    def test_bunch_of_slashes_in_path(self):
-        self.assertEqual("/target/my/path/",
-                         util.target_path("/target/", "//my/path/"))
-        self.assertEqual("/target/my/path/",
-                         util.target_path("/target/", "///my/path/"))
-
-    def test_c_lang_can_take_utf8_args(self):
-        """Independent of system LC_CTYPE, args can contain utf-8 strings.
-
-        When python starts up, its default encoding gets set based on
-        the value of LC_CTYPE.  If no system locale is set, the default
-        encoding for both python2 and python3 in some paths will end up
-        being ascii.
-
-        Attempts to use setlocale or patching (or changing) os.environ
-        in the current environment seem to not be effective.
-
-        This test starts up a python with LC_CTYPE set to C so that
-        the default encoding will be set to ascii.  In such an environment
-        Popen(['command', 'non-ascii-arg']) would cause a UnicodeDecodeError.
-        """
-        python_prog = '\n'.join([
-            'import json, sys',
-            'from cloudinit.util import subp',
-            'data = sys.stdin.read()',
-            'cmd = json.loads(data)',
-            'subp(cmd, capture=False)',
-            ''])
-        cmd = [BASH, '-c', 'echo -n "$@"', '--',
-               self.utf8_valid.decode("utf-8")]
-        python_subp = [sys.executable, '-c', python_prog]
-
-        out, _err = util.subp(
-            python_subp, update_env={'LC_CTYPE': 'C'},
-            data=json.dumps(cmd).encode("utf-8"),
-            decode=False)
-        self.assertEqual(self.utf8_valid, out)
-
-    def test_bogus_command_logs_status_messages(self):
-        """status_cb gets status messages logs on bogus commands provided."""
-        logs = []
-
-        def status_cb(log):
-            logs.append(log)
-
-        with self.assertRaises(util.ProcessExecutionError):
-            util.subp([BOGUS_COMMAND], status_cb=status_cb)
-
-        expected = [
-            'Begin run command: {cmd}\n'.format(cmd=BOGUS_COMMAND),
-            'ERROR: End run command: invalid command provided\n']
-        self.assertEqual(expected, logs)
-
-    def test_command_logs_exit_codes_to_status_cb(self):
-        """status_cb gets status messages containing command exit code."""
-        logs = []
-
-        def status_cb(log):
-            logs.append(log)
-
-        with self.assertRaises(util.ProcessExecutionError):
-            util.subp([BASH, '-c', 'exit 2'], status_cb=status_cb)
-        util.subp([BASH, '-c', 'exit 0'], status_cb=status_cb)
-
-        expected = [
-            'Begin run command: %s -c exit 2\n' % BASH,
-            'ERROR: End run command: exit(2)\n',
-            'Begin run command: %s -c exit 0\n' % BASH,
-            'End run command: exit(0)\n']
-        self.assertEqual(expected, logs)
-
-
 class TestEncode(helpers.TestCase):
     """Test the encoding functions"""
     def test_decode_binary_plain_text_with_hex(self):
@@ -966,7 +752,7 @@ class TestProcessExecutionError(helpers.TestCase):
     empty_description = 'Unexpected error while running command.'
 
     def test_pexec_error_indent_text(self):
-        error = util.ProcessExecutionError()
+        error = subp.ProcessExecutionError()
         msg = 'abc\ndef'
         formatted = 'abc\n{0}def'.format(' ' * 4)
         self.assertEqual(error._indent_text(msg, indent_level=4), formatted)
@@ -976,10 +762,10 @@ class TestProcessExecutionError(helpers.TestCase):
             error._indent_text(msg.encode()), type(msg.encode()))
 
     def test_pexec_error_type(self):
-        self.assertIsInstance(util.ProcessExecutionError(), IOError)
+        self.assertIsInstance(subp.ProcessExecutionError(), IOError)
 
     def test_pexec_error_empty_msgs(self):
-        error = util.ProcessExecutionError()
+        error = subp.ProcessExecutionError()
         self.assertTrue(all(attr == self.empty_attr for attr in
                             (error.stderr, error.stdout, error.reason)))
         self.assertEqual(error.description, self.empty_description)
@@ -993,7 +779,7 @@ class TestProcessExecutionError(helpers.TestCase):
         stderr_msg = 'error error'
         cmd = 'test command'
         exit_code = 3
-        error = util.ProcessExecutionError(
+        error = subp.ProcessExecutionError(
             stdout=stdout_msg, stderr=stderr_msg, exit_code=3, cmd=cmd)
         self.assertEqual(str(error), self.template.format(
             description=self.empty_description, stdout=stdout_msg,
@@ -1004,7 +790,7 @@ class TestProcessExecutionError(helpers.TestCase):
         # make sure bytes is converted handled properly when formatting
         stdout_msg = 'multi\nline\noutput message'.encode()
         stderr_msg = 'multi\nline\nerror message\n\n\n'
-        error = util.ProcessExecutionError(
+        error = subp.ProcessExecutionError(
             stdout=stdout_msg, stderr=stderr_msg)
         self.assertEqual(
             str(error),
@@ -1170,7 +956,7 @@ class TestGetProcEnv(helpers.TestCase):
         self.assertEqual(my_ppid, util.get_proc_ppid(my_pid))
 
 
-@mock.patch('cloudinit.util.subp')
+@mock.patch('cloudinit.subp.subp')
 def test_find_devs_with_openbsd(m_subp):
     m_subp.return_value = (
         'cd0:,sd0:630d98d32b5d3759,sd1:,fd0:', ''
@@ -1179,7 +965,7 @@ def test_find_devs_with_openbsd(m_subp):
     assert devlist == ['/dev/cd0a', '/dev/sd1i']
 
 
-@mock.patch('cloudinit.util.subp')
+@mock.patch('cloudinit.subp.subp')
 def test_find_devs_with_openbsd_with_criteria(m_subp):
     m_subp.return_value = (
         'cd0:,sd0:630d98d32b5d3759,sd1:,fd0:', ''
@@ -1209,7 +995,7 @@ def test_find_devs_with_freebsd(m_glob):
     assert devlist == ['/dev/msdosfs/EFISYS']
 
 
-@mock.patch("cloudinit.util.subp")
+@mock.patch("cloudinit.subp.subp")
 def test_find_devs_with_netbsd(m_subp):
     side_effect_values = [
         ("ld0 dk0 dk1 cd0", ""),
diff --git a/tests/unittests/test_vmware/test_guestcust_util.py b/tests/unittests/test_vmware/test_guestcust_util.py
index 394bee9f..c8b59d83 100644
--- a/tests/unittests/test_vmware/test_guestcust_util.py
+++ b/tests/unittests/test_vmware/test_guestcust_util.py
@@ -5,7 +5,7 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from cloudinit import util
+from cloudinit import subp
 from cloudinit.sources.helpers.vmware.imc.config import Config
 from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile
 from cloudinit.sources.helpers.vmware.imc.guestcust_util import (
@@ -21,7 +21,7 @@ class TestGuestCustUtil(CiTestCase):
         This test is designed to verify the behavior if vmware-toolbox-cmd
         is not installed.
         """
-        with mock.patch.object(util, 'which', return_value=None):
+        with mock.patch.object(subp, 'which', return_value=None):
             self.assertEqual(
                 get_tools_config('section', 'key', 'defaultVal'), 'defaultVal')
 
@@ -30,10 +30,10 @@ class TestGuestCustUtil(CiTestCase):
         This test is designed to verify the behavior if internal exception
         is raised.
         """
-        with mock.patch.object(util, 'which', return_value='/dummy/path'):
-            with mock.patch.object(util, 'subp',
+        with mock.patch.object(subp, 'which', return_value='/dummy/path'):
+            with mock.patch.object(subp, 'subp',
                                    return_value=('key=value', b''),
-                                   side_effect=util.ProcessExecutionError(
+                                   side_effect=subp.ProcessExecutionError(
                                        "subp failed", exit_code=99)):
                 # verify return value is 'defaultVal', not 'value'.
                 self.assertEqual(
@@ -45,28 +45,28 @@ class TestGuestCustUtil(CiTestCase):
         This test is designed to verify the value could be parsed from
         key = value of the given [section]
         """
-        with mock.patch.object(util, 'which', return_value='/dummy/path'):
+        with mock.patch.object(subp, 'which', return_value='/dummy/path'):
             # value is not blank
-            with mock.patch.object(util, 'subp',
+            with mock.patch.object(subp, 'subp',
                                    return_value=('key =   value  ', b'')):
                 self.assertEqual(
                     get_tools_config('section', 'key', 'defaultVal'),
                     'value')
             # value is blank
-            with mock.patch.object(util, 'subp',
+            with mock.patch.object(subp, 'subp',
                                    return_value=('key = ', b'')):
                 self.assertEqual(
                     get_tools_config('section', 'key', 'defaultVal'),
                     '')
             # value contains =
-            with mock.patch.object(util, 'subp',
+            with mock.patch.object(subp, 'subp',
                                    return_value=('key=Bar=Wark', b'')):
                 self.assertEqual(
                     get_tools_config('section', 'key', 'defaultVal'),
                     'Bar=Wark')
 
             # value contains specific characters
-            with mock.patch.object(util, 'subp',
+            with mock.patch.object(subp, 'subp',
                                    return_value=('[a] b.c_d=e-f', b'')):
                 self.assertEqual(
                     get_tools_config('section', 'key', 'defaultVal'),
@@ -87,7 +87,7 @@ class TestGuestCustUtil(CiTestCase):
         # post gc status is YES, subp is called to execute command
         cf._insertKey("MISC|POST-GC-STATUS", "YES")
         conf = Config(cf)
-        with mock.patch.object(util, 'subp',
+        with mock.patch.object(subp, 'subp',
                                return_value=('ok', b'')) as mockobj:
             self.assertEqual(
                 set_gc_status(conf, 'Successful'), ('ok', b''))
diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers
index c3113705..01916b70 100644
--- a/tools/.github-cla-signers
+++ b/tools/.github-cla-signers
@@ -4,5 +4,6 @@ dhensby
 lucasmoura
 matthewruffell
 nishigori
+smoser
 tomponline
 TheRealFalcon
-- 
cgit v1.2.3


From f3bd42659efeed4b092ffcdfd5df7f24813f2d3e Mon Sep 17 00:00:00 2001
From: Joshua Powers <josh.powers@canonical.com>
Date: Wed, 10 Jun 2020 07:39:29 -0700
Subject: test: fix all flake8 E126 errors (#425)

---
 cloudinit/cmd/devel/render.py                      |   5 +-
 cloudinit/cmd/query.py                             |   5 +-
 cloudinit/distros/netbsd.py                        |  10 +-
 cloudinit/distros/openbsd.py                       |   7 +-
 cloudinit/net/__init__.py                          |   5 +-
 cloudinit/net/bsd.py                               |   5 +-
 cloudinit/net/netbsd.py                            |   5 +-
 cloudinit/net/openbsd.py                           |   5 +-
 cloudinit/net/sysconfig.py                         |   2 +-
 cloudinit/reporting/handlers.py                    |  14 +-
 cloudinit/sources/DataSourceAzure.py               |  33 +++--
 cloudinit/sources/helpers/tests/test_netlink.py    | 165 +++++++++++++--------
 cloudinit/sources/tests/test_init.py               |   9 +-
 tests/cloud_tests/testcases/base.py                |   2 +-
 tests/unittests/test_datasource/test_azure.py      |  33 +++--
 tests/unittests/test_datasource/test_scaleway.py   |  42 ++++--
 tests/unittests/test_distros/test_bsd_utils.py     |   5 +-
 .../unittests/test_handler/test_handler_mounts.py  |   7 +-
 tests/unittests/test_reporting_hyperv.py           |  11 +-
 tests/unittests/test_sshutil.py                    |  18 ++-
 tox.ini                                            |   3 +-
 21 files changed, 238 insertions(+), 153 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/cloudinit/cmd/devel/render.py b/cloudinit/cmd/devel/render.py
index 1bc22406..1090aa16 100755
--- a/cloudinit/cmd/devel/render.py
+++ b/cloudinit/cmd/devel/render.py
@@ -57,8 +57,9 @@ def handle_args(name, args):
                 paths.run_dir, INSTANCE_JSON_SENSITIVE_FILE)
             if not os.path.exists(instance_data_fn):
                 LOG.warning(
-                     'Missing root-readable %s. Using redacted %s instead.',
-                     instance_data_fn, redacted_data_fn)
+                    'Missing root-readable %s. Using redacted %s instead.',
+                    instance_data_fn, redacted_data_fn
+                )
                 instance_data_fn = redacted_data_fn
         else:
             instance_data_fn = redacted_data_fn
diff --git a/cloudinit/cmd/query.py b/cloudinit/cmd/query.py
index e3db8679..0fb48ebd 100644
--- a/cloudinit/cmd/query.py
+++ b/cloudinit/cmd/query.py
@@ -90,8 +90,9 @@ def handle_args(name, args):
                 instance_data_fn = sensitive_data_fn
             else:
                 LOG.warning(
-                     'Missing root-readable %s. Using redacted %s instead.',
-                     sensitive_data_fn, redacted_data_fn)
+                    'Missing root-readable %s. Using redacted %s instead.',
+                    sensitive_data_fn, redacted_data_fn
+                )
                 instance_data_fn = redacted_data_fn
         else:
             instance_data_fn = redacted_data_fn
diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py
index 066737a8..f1a9b182 100644
--- a/cloudinit/distros/netbsd.py
+++ b/cloudinit/distros/netbsd.py
@@ -100,8 +100,9 @@ class NetBSD(cloudinit.distros.bsd.BSD):
         else:
             method = crypt.METHOD_BLOWFISH  # pylint: disable=E1101
             hashed_pw = crypt.crypt(
-                    passwd,
-                    crypt.mksalt(method))
+                passwd,
+                crypt.mksalt(method)
+            )
 
         try:
             subp.subp(['usermod', '-p', hashed_pw, user])
@@ -143,8 +144,9 @@ class NetBSD(cloudinit.distros.bsd.BSD):
         os_arch = platform.machine()
         e = os.environ.copy()
         e['PKG_PATH'] = (
-                'http://cdn.netbsd.org/pub/pkgsrc/'
-                'packages/NetBSD/%s/%s/All') % (os_arch, os_release)
+            'http://cdn.netbsd.org/pub/pkgsrc/'
+            'packages/NetBSD/%s/%s/All'
+        ) % (os_arch, os_release)
         return e
 
     def update_package_sources(self):
diff --git a/cloudinit/distros/openbsd.py b/cloudinit/distros/openbsd.py
index 07c76530..720c9cf3 100644
--- a/cloudinit/distros/openbsd.py
+++ b/cloudinit/distros/openbsd.py
@@ -42,9 +42,10 @@ class Distro(cloudinit.distros.netbsd.NetBSD):
         os_arch = platform.machine()
         e = os.environ.copy()
         e['PKG_PATH'] = (
-                'ftp://ftp.openbsd.org/pub/OpenBSD/{os_release}/'
-                'packages/{os_arch}/').format(
-                        os_arch=os_arch, os_release=os_release)
+            'ftp://ftp.openbsd.org/pub/OpenBSD/{os_release}/'
+            'packages/{os_arch}/').format(
+            os_arch=os_arch, os_release=os_release
+        )
         return e
 
 
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index a57fea0a..b40cb154 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -849,8 +849,9 @@ def get_interfaces_by_mac_on_freebsd():
 def get_interfaces_by_mac_on_netbsd():
     ret = {}
     re_field_match = (
-            r"(?P<ifname>\w+).*address:\s"
-            r"(?P<mac>([\da-f]{2}[:-]){5}([\da-f]{2})).*")
+        r"(?P<ifname>\w+).*address:\s"
+        r"(?P<mac>([\da-f]{2}[:-]){5}([\da-f]{2})).*"
+    )
     (out, _) = subp.subp(['ifconfig', '-a'])
     if_lines = re.sub(r'\n\s+', ' ', out).splitlines()
     for line in if_lines:
diff --git a/cloudinit/net/bsd.py b/cloudinit/net/bsd.py
index 1c355a98..e34e0454 100644
--- a/cloudinit/net/bsd.py
+++ b/cloudinit/net/bsd.py
@@ -66,8 +66,9 @@ class BSDRenderer(renderer.Renderer):
                 if subnet.get('type') == 'static':
                     if not subnet.get('netmask'):
                         LOG.debug(
-                                'Skipping IP %s, because there is no netmask',
-                                subnet.get('address'))
+                            'Skipping IP %s, because there is no netmask',
+                            subnet.get('address')
+                        )
                         continue
                     LOG.debug('Configuring dev %s with %s / %s', device_name,
                               subnet.get('address'), subnet.get('netmask'))
diff --git a/cloudinit/net/netbsd.py b/cloudinit/net/netbsd.py
index 30437b5f..71b38ee6 100644
--- a/cloudinit/net/netbsd.py
+++ b/cloudinit/net/netbsd.py
@@ -17,8 +17,9 @@ class Renderer(cloudinit.net.bsd.BSDRenderer):
         if self.dhcp_interfaces():
             self.set_rc_config_value('dhcpcd', 'YES')
             self.set_rc_config_value(
-                    'dhcpcd_flags',
-                    ' '.join(self.dhcp_interfaces()))
+                'dhcpcd_flags',
+                ' '.join(self.dhcp_interfaces())
+            )
         for device_name, v in self.interface_configurations.items():
             if isinstance(v, dict):
                 self.set_rc_config_value(
diff --git a/cloudinit/net/openbsd.py b/cloudinit/net/openbsd.py
index 489ea48b..166d77e6 100644
--- a/cloudinit/net/openbsd.py
+++ b/cloudinit/net/openbsd.py
@@ -19,8 +19,9 @@ class Renderer(cloudinit.net.bsd.BSDRenderer):
             elif isinstance(v, dict):
                 try:
                     content = "inet {address} {netmask}\n".format(
-                            address=v['address'],
-                            netmask=v['netmask'])
+                        address=v['address'],
+                        netmask=v['netmask']
+                    )
                 except KeyError:
                     LOG.error(
                         "Invalid static configuration for %s",
diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py
index f36c300f..0a5d481d 100644
--- a/cloudinit/net/sysconfig.py
+++ b/cloudinit/net/sysconfig.py
@@ -505,7 +505,7 @@ class Renderer(renderer.Renderer):
                             iface_cfg['IPADDR6_%d' % ipv6_index] = ipv6_cidr
                         else:
                             iface_cfg['IPV6ADDR_SECONDARIES'] += \
-                                                        " " + ipv6_cidr
+                                " " + ipv6_cidr
                 else:
                     ipv4_index = ipv4_index + 1
                     suff = "" if ipv4_index == 0 else str(ipv4_index)
diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py
index 00e8d2e5..6b9127b6 100755
--- a/cloudinit/reporting/handlers.py
+++ b/cloudinit/reporting/handlers.py
@@ -139,7 +139,8 @@ class HyperVKvpReportingHandler(ReportingHandler):
         self.event_key_prefix = u"{0}|{1}".format(self.EVENT_PREFIX,
                                                   self.incarnation_no)
         self.publish_thread = threading.Thread(
-                target=self._publish_event_routine)
+            target=self._publish_event_routine
+        )
         self.publish_thread.daemon = True
         self.publish_thread.start()
 
@@ -202,10 +203,15 @@ class HyperVKvpReportingHandler(ReportingHandler):
                                          uuid.uuid4())
 
     def _encode_kvp_item(self, key, value):
-        data = (struct.pack("%ds%ds" % (
+        data = struct.pack(
+            "%ds%ds"
+            % (
                 self.HV_KVP_EXCHANGE_MAX_KEY_SIZE,
-                self.HV_KVP_EXCHANGE_MAX_VALUE_SIZE),
-                key.encode('utf-8'), value.encode('utf-8')))
+                self.HV_KVP_EXCHANGE_MAX_VALUE_SIZE,
+            ),
+            key.encode("utf-8"),
+            value.encode("utf-8"),
+        )
         return data
 
     def _decode_kvp_item(self, record_data):
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 89312b9e..6d569057 100755
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -523,8 +523,9 @@ class DataSourceAzure(sources.DataSource):
 
         try:
             crawled_data = util.log_time(
-                        logfunc=LOG.debug, msg='Crawl of metadata service',
-                        func=self.crawl_metadata)
+                logfunc=LOG.debug, msg='Crawl of metadata service',
+                func=self.crawl_metadata
+            )
         except sources.InvalidMetaDataException as e:
             LOG.warning('Could not crawl Azure metadata: %s', e)
             return False
@@ -893,9 +894,10 @@ def can_dev_be_reformatted(devpath, preserve_ntfs):
             (cand_part, cand_path, devpath))
 
     with events.ReportEventStack(
-                name="mount-ntfs-and-count",
-                description="mount-ntfs-and-count",
-                parent=azure_ds_reporter) as evt:
+        name="mount-ntfs-and-count",
+        description="mount-ntfs-and-count",
+        parent=azure_ds_reporter
+    ) as evt:
         try:
             file_count = util.mount_cb(cand_path, count_files, mtype="ntfs",
                                        update_env_for_mount={'LANG': 'C'})
@@ -924,9 +926,10 @@ def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, maxwait=120,
     # wait for ephemeral disk to come up
     naplen = .2
     with events.ReportEventStack(
-                name="wait-for-ephemeral-disk",
-                description="wait for ephemeral disk",
-                parent=azure_ds_reporter):
+        name="wait-for-ephemeral-disk",
+        description="wait for ephemeral disk",
+        parent=azure_ds_reporter
+    ):
         missing = util.wait_for_files([devpath],
                                       maxwait=maxwait,
                                       naplen=naplen,
@@ -1334,9 +1337,10 @@ def parse_network_config(imds_metadata):
     @return: Dictionary containing network version 2 standard configuration.
     """
     with events.ReportEventStack(
-                name="parse_network_config",
-                description="",
-                parent=azure_ds_reporter) as evt:
+        name="parse_network_config",
+        description="",
+        parent=azure_ds_reporter
+    ) as evt:
         if imds_metadata != sources.UNSET and imds_metadata:
             netconfig = {'version': 2, 'ethernets': {}}
             LOG.debug('Azure: generating network configuration from IMDS')
@@ -1480,9 +1484,10 @@ def maybe_remove_ubuntu_network_config_scripts(paths=None):
 
 def _is_platform_viable(seed_dir):
     with events.ReportEventStack(
-                name="check-platform-viability",
-                description="found azure asset tag",
-                parent=azure_ds_reporter) as evt:
+        name="check-platform-viability",
+        description="found azure asset tag",
+        parent=azure_ds_reporter
+    ) as evt:
 
         """Check platform environment to report if this datasource may run."""
         asset_tag = util.read_dmi_data('chassis-asset-tag')
diff --git a/cloudinit/sources/helpers/tests/test_netlink.py b/cloudinit/sources/helpers/tests/test_netlink.py
index 58c3adc6..10760bd6 100644
--- a/cloudinit/sources/helpers/tests/test_netlink.py
+++ b/cloudinit/sources/helpers/tests/test_netlink.py
@@ -180,17 +180,22 @@ class TestWaitForMediaDisconnectConnect(CiTestCase):
         other_ifname = "eth1"
         expected_ifname = "eth0"
         data_op_down_eth1 = self._media_switch_data(
-                                other_ifname, RTM_NEWLINK, OPER_DOWN)
+            other_ifname, RTM_NEWLINK, OPER_DOWN
+        )
         data_op_up_eth1 = self._media_switch_data(
-                                other_ifname, RTM_NEWLINK, OPER_UP)
+            other_ifname, RTM_NEWLINK, OPER_UP
+        )
         data_op_down_eth0 = self._media_switch_data(
-                                expected_ifname, RTM_NEWLINK, OPER_DOWN)
+            expected_ifname, RTM_NEWLINK, OPER_DOWN
+        )
         data_op_up_eth0 = self._media_switch_data(
-                                expected_ifname, RTM_NEWLINK, OPER_UP)
-        m_read_netlink_socket.side_effect = [data_op_down_eth1,
-                                             data_op_up_eth1,
-                                             data_op_down_eth0,
-                                             data_op_up_eth0]
+            expected_ifname, RTM_NEWLINK, OPER_UP)
+        m_read_netlink_socket.side_effect = [
+            data_op_down_eth1,
+            data_op_up_eth1,
+            data_op_down_eth0,
+            data_op_up_eth0
+        ]
         wait_for_media_disconnect_connect(m_socket, expected_ifname)
         self.assertIn('Ignored netlink event on interface %s' % other_ifname,
                       self.logs.getvalue())
@@ -207,17 +212,23 @@ class TestWaitForMediaDisconnectConnect(CiTestCase):
         '''
         ifname = "eth0"
         data_getlink_down = self._media_switch_data(
-                                    ifname, RTM_GETLINK, OPER_DOWN)
+            ifname, RTM_GETLINK, OPER_DOWN
+        )
         data_getlink_up = self._media_switch_data(
-                                    ifname, RTM_GETLINK, OPER_UP)
+            ifname, RTM_GETLINK, OPER_UP
+        )
         data_newlink_down = self._media_switch_data(
-                                    ifname, RTM_NEWLINK, OPER_DOWN)
+            ifname, RTM_NEWLINK, OPER_DOWN
+        )
         data_newlink_up = self._media_switch_data(
-                                    ifname, RTM_NEWLINK, OPER_UP)
-        m_read_netlink_socket.side_effect = [data_getlink_down,
-                                             data_getlink_up,
-                                             data_newlink_down,
-                                             data_newlink_up]
+            ifname, RTM_NEWLINK, OPER_UP
+        )
+        m_read_netlink_socket.side_effect = [
+            data_getlink_down,
+            data_getlink_up,
+            data_newlink_down,
+            data_newlink_up
+        ]
         wait_for_media_disconnect_connect(m_socket, ifname)
         self.assertEqual(m_read_netlink_socket.call_count, 4)
 
@@ -233,19 +244,25 @@ class TestWaitForMediaDisconnectConnect(CiTestCase):
         '''
         ifname = "eth0"
         data_setlink_down = self._media_switch_data(
-                                    ifname, RTM_SETLINK, OPER_DOWN)
+            ifname, RTM_SETLINK, OPER_DOWN
+        )
         data_setlink_up = self._media_switch_data(
-                                    ifname, RTM_SETLINK, OPER_UP)
+            ifname, RTM_SETLINK, OPER_UP
+        )
         data_newlink_down = self._media_switch_data(
-                                    ifname, RTM_NEWLINK, OPER_DOWN)
+            ifname, RTM_NEWLINK, OPER_DOWN
+        )
         data_newlink_up = self._media_switch_data(
-                                    ifname, RTM_NEWLINK, OPER_UP)
-        m_read_netlink_socket.side_effect = [data_setlink_down,
-                                             data_setlink_up,
-                                             data_newlink_down,
-                                             data_newlink_up,
-                                             data_newlink_down,
-                                             data_newlink_up]
+            ifname, RTM_NEWLINK, OPER_UP
+        )
+        m_read_netlink_socket.side_effect = [
+            data_setlink_down,
+            data_setlink_up,
+            data_newlink_down,
+            data_newlink_up,
+            data_newlink_down,
+            data_newlink_up
+        ]
         wait_for_media_disconnect_connect(m_socket, ifname)
         self.assertEqual(m_read_netlink_socket.call_count, 4)
 
@@ -255,23 +272,30 @@ class TestWaitForMediaDisconnectConnect(CiTestCase):
         ifname = "eth0"
         data_op_down = self._media_switch_data(ifname, RTM_NEWLINK, OPER_DOWN)
         data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP)
-        data_op_dormant = self._media_switch_data(ifname, RTM_NEWLINK,
-                                                  OPER_DORMANT)
-        data_op_notpresent = self._media_switch_data(ifname, RTM_NEWLINK,
-                                                     OPER_NOTPRESENT)
-        data_op_lowerdown = self._media_switch_data(ifname, RTM_NEWLINK,
-                                                    OPER_LOWERLAYERDOWN)
-        data_op_testing = self._media_switch_data(ifname, RTM_NEWLINK,
-                                                  OPER_TESTING)
-        data_op_unknown = self._media_switch_data(ifname, RTM_NEWLINK,
-                                                  OPER_UNKNOWN)
-        m_read_netlink_socket.side_effect = [data_op_up, data_op_up,
-                                             data_op_dormant, data_op_up,
-                                             data_op_notpresent, data_op_up,
-                                             data_op_lowerdown, data_op_up,
-                                             data_op_testing, data_op_up,
-                                             data_op_unknown, data_op_up,
-                                             data_op_down, data_op_up]
+        data_op_dormant = self._media_switch_data(
+            ifname, RTM_NEWLINK, OPER_DORMANT
+        )
+        data_op_notpresent = self._media_switch_data(
+            ifname, RTM_NEWLINK, OPER_NOTPRESENT
+        )
+        data_op_lowerdown = self._media_switch_data(
+            ifname, RTM_NEWLINK, OPER_LOWERLAYERDOWN
+        )
+        data_op_testing = self._media_switch_data(
+            ifname, RTM_NEWLINK, OPER_TESTING
+        )
+        data_op_unknown = self._media_switch_data(
+            ifname, RTM_NEWLINK, OPER_UNKNOWN
+        )
+        m_read_netlink_socket.side_effect = [
+            data_op_up, data_op_up,
+            data_op_dormant, data_op_up,
+            data_op_notpresent, data_op_up,
+            data_op_lowerdown, data_op_up,
+            data_op_testing, data_op_up,
+            data_op_unknown, data_op_up,
+            data_op_down, data_op_up
+        ]
         wait_for_media_disconnect_connect(m_socket, ifname)
         self.assertEqual(m_read_netlink_socket.call_count, 14)
 
@@ -281,12 +305,14 @@ class TestWaitForMediaDisconnectConnect(CiTestCase):
         ifname = "eth0"
         data_op_down = self._media_switch_data(ifname, RTM_NEWLINK, OPER_DOWN)
         data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP)
-        data_op_dormant = self._media_switch_data(ifname, RTM_NEWLINK,
-                                                  OPER_DORMANT)
-        data_op_unknown = self._media_switch_data(ifname, RTM_NEWLINK,
-                                                  OPER_UNKNOWN)
-        m_read_netlink_socket.side_effect = [data_op_down, data_op_dormant,
-                                             data_op_unknown, data_op_up]
+        data_op_dormant = self._media_switch_data(
+            ifname, RTM_NEWLINK, OPER_DORMANT)
+        data_op_unknown = self._media_switch_data(
+            ifname, RTM_NEWLINK, OPER_UNKNOWN)
+        m_read_netlink_socket.side_effect = [
+            data_op_down, data_op_dormant,
+            data_op_unknown, data_op_up
+        ]
         wait_for_media_disconnect_connect(m_socket, ifname)
         self.assertEqual(m_read_netlink_socket.call_count, 4)
 
@@ -300,9 +326,11 @@ class TestWaitForMediaDisconnectConnect(CiTestCase):
         data_op_down = self._media_switch_data(ifname, RTM_NEWLINK, OPER_DOWN)
         data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP)
         data_op_invalid = self._media_switch_data(ifname, RTM_NEWLINK, 7)
-        m_read_netlink_socket.side_effect = [data_op_invalid, data_op_up,
-                                             data_op_down, data_op_invalid,
-                                             data_op_up]
+        m_read_netlink_socket.side_effect = [
+            data_op_invalid, data_op_up,
+            data_op_down, data_op_invalid,
+            data_op_up
+        ]
         wait_for_media_disconnect_connect(m_socket, ifname)
         self.assertEqual(m_read_netlink_socket.call_count, 5)
 
@@ -333,8 +361,9 @@ class TestWaitForMediaDisconnectConnect(CiTestCase):
         data_invalid2 = self._media_switch_data(ifname, RTM_NEWLINK, None)
         data_op_down = self._media_switch_data(ifname, RTM_NEWLINK, OPER_DOWN)
         data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP)
-        m_read_netlink_socket.side_effect = [data_invalid1, data_invalid2,
-                                             data_op_down, data_op_up]
+        m_read_netlink_socket.side_effect = [
+            data_invalid1, data_invalid2, data_op_down, data_op_up
+        ]
         wait_for_media_disconnect_connect(m_socket, ifname)
         self.assertEqual(m_read_netlink_socket.call_count, 4)
 
@@ -344,11 +373,15 @@ class TestWaitForMediaDisconnectConnect(CiTestCase):
         bytes = ifname.encode("utf-8")
         data = bytearray(96)
         struct.pack_into("=LHHLL", data, 0, 48, RTM_NEWLINK, 0, 0, 0)
-        struct.pack_into("HH4sHHc", data, RTATTR_START_OFFSET, 8, 3,
-                         bytes, 5, 16, int_to_bytes(OPER_DOWN))
+        struct.pack_into(
+            "HH4sHHc", data, RTATTR_START_OFFSET, 8, 3,
+            bytes, 5, 16, int_to_bytes(OPER_DOWN)
+        )
         struct.pack_into("=LHHLL", data, 48, 48, RTM_NEWLINK, 0, 0, 0)
-        struct.pack_into("HH4sHHc", data, 48 + RTATTR_START_OFFSET, 8,
-                         3, bytes, 5, 16, int_to_bytes(OPER_UP))
+        struct.pack_into(
+            "HH4sHHc", data, 48 + RTATTR_START_OFFSET, 8,
+            3, bytes, 5, 16, int_to_bytes(OPER_UP)
+        )
         m_read_netlink_socket.return_value = data
         wait_for_media_disconnect_connect(m_socket, ifname)
         self.assertEqual(m_read_netlink_socket.call_count, 1)
@@ -360,14 +393,18 @@ class TestWaitForMediaDisconnectConnect(CiTestCase):
         data1 = bytearray(112)
         data2 = bytearray(32)
         struct.pack_into("=LHHLL", data1, 0, 48, RTM_NEWLINK, 0, 0, 0)
-        struct.pack_into("HH4sHHc", data1, RTATTR_START_OFFSET, 8, 3,
-                         bytes, 5, 16, int_to_bytes(OPER_DOWN))
+        struct.pack_into(
+            "HH4sHHc", data1, RTATTR_START_OFFSET, 8, 3,
+            bytes, 5, 16, int_to_bytes(OPER_DOWN)
+        )
         struct.pack_into("=LHHLL", data1, 48, 48, RTM_NEWLINK, 0, 0, 0)
-        struct.pack_into("HH4sHHc", data1, 80, 8, 3, bytes, 5, 16,
-                         int_to_bytes(OPER_DOWN))
+        struct.pack_into(
+            "HH4sHHc", data1, 80, 8, 3, bytes, 5, 16, int_to_bytes(OPER_DOWN)
+        )
         struct.pack_into("=LHHLL", data1, 96, 48, RTM_NEWLINK, 0, 0, 0)
-        struct.pack_into("HH4sHHc", data2, 16, 8, 3, bytes, 5, 16,
-                         int_to_bytes(OPER_UP))
+        struct.pack_into(
+            "HH4sHHc", data2, 16, 8, 3, bytes, 5, 16, int_to_bytes(OPER_UP)
+        )
         m_read_netlink_socket.side_effect = [data1, data2]
         wait_for_media_disconnect_connect(m_socket, ifname)
         self.assertEqual(m_read_netlink_socket.call_count, 2)
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index 5b6f1b3f..1420a988 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -436,10 +436,11 @@ class TestDataSource(CiTestCase):
         expected = {
             'base64_encoded_keys': [],
             'merged_cfg': {
-                 '_doc': (
-                     'Merged cloud-init system config from '
-                     '/etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/'),
-                 'datasource': {'_undef': {'key1': False}}},
+                '_doc': (
+                    'Merged cloud-init system config from '
+                    '/etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/'
+                ),
+                'datasource': {'_undef': {'key1': False}}},
             'sensitive_keys': [
                 'ds/meta_data/some/security-credentials', 'merged_cfg'],
             'sys_info': sys_info,
diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index 68d59111..2e7c6686 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -321,7 +321,7 @@ class CloudTestCase(unittest.TestCase):
             "Unexpected sys_info dist value")
         self.assertEqual(self.os_name, v1_data['distro_release'])
         self.assertEqual(
-                str(self.os_cfg['version']), v1_data['distro_version'])
+            str(self.os_cfg['version']), v1_data['distro_version'])
         self.assertEqual('x86_64', v1_data['machine'])
         self.assertIsNotNone(
             re.match(r'3.\d\.\d', v1_data['python_version']),
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 05552a1e..a99cbd41 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -683,15 +683,17 @@ scbus-1 on xpt0 bus 0
         'cloudinit.sources.DataSourceAzure.DataSourceAzure._report_ready')
     @mock.patch('cloudinit.sources.DataSourceAzure.DataSourceAzure._poll_imds')
     def test_crawl_metadata_on_reprovision_reports_ready(
-                            self, poll_imds_func,
-                            report_ready_func,
-                            m_write, m_dhcp):
+        self, poll_imds_func, report_ready_func, m_write, m_dhcp
+    ):
         """If reprovisioning, report ready at the end"""
         ovfenv = construct_valid_ovf_env(
-                            platform_settings={"PreprovisionedVm": "True"})
+            platform_settings={"PreprovisionedVm": "True"}
+        )
 
-        data = {'ovfcontent': ovfenv,
-                'sys_cfg': {}}
+        data = {
+            'ovfcontent': ovfenv,
+            'sys_cfg': {}
+        }
         dsrc = self._get_ds(data)
         poll_imds_func.return_value = ovfenv
         dsrc.crawl_metadata()
@@ -706,15 +708,18 @@ scbus-1 on xpt0 bus 0
     @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
     @mock.patch('cloudinit.sources.DataSourceAzure.readurl')
     def test_crawl_metadata_on_reprovision_reports_ready_using_lease(
-                            self, m_readurl, m_dhcp,
-                            m_net, report_ready_func,
-                            m_media_switch, m_write):
+        self, m_readurl, m_dhcp, m_net, report_ready_func,
+        m_media_switch, m_write
+    ):
         """If reprovisioning, report ready using the obtained lease"""
         ovfenv = construct_valid_ovf_env(
-                            platform_settings={"PreprovisionedVm": "True"})
+            platform_settings={"PreprovisionedVm": "True"}
+        )
 
-        data = {'ovfcontent': ovfenv,
-                'sys_cfg': {}}
+        data = {
+            'ovfcontent': ovfenv,
+            'sys_cfg': {}
+        }
         dsrc = self._get_ds(data)
 
         lease = {
@@ -1955,8 +1960,8 @@ class TestPreprovisioningPollIMDS(CiTestCase):
                 response = requests.Response()
                 response.status_code = 404 if self.tries == 2 else 410
                 raise requests.exceptions.HTTPError(
-                        "fake {}".format(response.status_code),
-                        response=response)
+                    "fake {}".format(response.status_code), response=response
+                )
             # Third try should succeed and stop retries or redhcp
             return mock.MagicMock(status_code=200, text="good", content="good")
 
diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py
index 15441454..9d82bda9 100644
--- a/tests/unittests/test_datasource/test_scaleway.py
+++ b/tests/unittests/test_datasource/test_scaleway.py
@@ -353,12 +353,16 @@ class TestDataSourceScaleway(HttprettyTestCase):
         self.datasource.metadata['ipv6'] = None
 
         netcfg = self.datasource.network_config
-        resp = {'version': 1,
-                'config': [{
-                     'type': 'physical',
-                     'name': 'ens2',
-                     'subnets': [{'type': 'dhcp4'}]}]
+        resp = {
+            'version': 1,
+            'config': [
+                {
+                    'type': 'physical',
+                    'name': 'ens2',
+                    'subnets': [{'type': 'dhcp4'}]
                 }
+            ]
+        }
         self.assertEqual(netcfg, resp)
 
     @mock.patch('cloudinit.sources.DataSourceScaleway.net.find_fallback_nic')
@@ -424,12 +428,16 @@ class TestDataSourceScaleway(HttprettyTestCase):
         self.datasource.metadata['ipv6'] = None
         self.datasource._network_config = sources.UNSET
 
-        resp = {'version': 1,
-                'config': [{
-                     'type': 'physical',
-                     'name': 'ens2',
-                     'subnets': [{'type': 'dhcp4'}]}]
+        resp = {
+            'version': 1,
+            'config': [
+                {
+                    'type': 'physical',
+                    'name': 'ens2',
+                    'subnets': [{'type': 'dhcp4'}]
                 }
+            ]
+        }
 
         netcfg = self.datasource.network_config
         self.assertEqual(netcfg, resp)
@@ -448,12 +456,16 @@ class TestDataSourceScaleway(HttprettyTestCase):
         self.datasource.metadata['ipv6'] = None
         self.datasource._network_config = None
 
-        resp = {'version': 1,
-                'config': [{
-                     'type': 'physical',
-                     'name': 'ens2',
-                     'subnets': [{'type': 'dhcp4'}]}]
+        resp = {
+            'version': 1,
+            'config': [
+                {
+                    'type': 'physical',
+                    'name': 'ens2',
+                    'subnets': [{'type': 'dhcp4'}]
                 }
+            ]
+        }
 
         netcfg = self.datasource.network_config
         self.assertEqual(netcfg, resp)
diff --git a/tests/unittests/test_distros/test_bsd_utils.py b/tests/unittests/test_distros/test_bsd_utils.py
index b38e4af5..3a68f2a9 100644
--- a/tests/unittests/test_distros/test_bsd_utils.py
+++ b/tests/unittests/test_distros/test_bsd_utils.py
@@ -62,5 +62,6 @@ class TestBsdUtils(CiTestCase):
         self.load_file.return_value = RC_FILE.format(hostname='foo')
         bsd_utils.set_rc_config_value('hostname', 'bar')
         self.write_file.assert_called_with(
-                '/etc/rc.conf',
-                RC_FILE.format(hostname='bar'))
+            '/etc/rc.conf',
+            RC_FILE.format(hostname='bar')
+        )
diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py
index 80c53c83..b643e3ae 100644
--- a/tests/unittests/test_handler/test_handler_mounts.py
+++ b/tests/unittests/test_handler/test_handler_mounts.py
@@ -260,8 +260,11 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase):
             '/dev/vdb /mnt auto defaults,noexec,comment=cloudconfig 0 2\n'
         )
         fstab_expected_content = fstab_original_content
-        cc = {'mounts': [
-                 ['/dev/vdb', '/mnt', 'auto', 'defaults,noexec']]}
+        cc = {
+            'mounts': [
+                ['/dev/vdb', '/mnt', 'auto', 'defaults,noexec']
+            ]
+        }
         with open(cc_mounts.FSTAB_PATH, 'w') as fd:
             fd.write(fstab_original_content)
         with open(cc_mounts.FSTAB_PATH, 'r') as fd:
diff --git a/tests/unittests/test_reporting_hyperv.py b/tests/unittests/test_reporting_hyperv.py
index fa8f8859..b60a66ab 100644
--- a/tests/unittests/test_reporting_hyperv.py
+++ b/tests/unittests/test_reporting_hyperv.py
@@ -93,10 +93,15 @@ class TextKvpReporter(CiTestCase):
     def test_not_truncate_kvp_file_modified_after_boot(self):
         with open(self.tmp_file_path, "wb+") as f:
             kvp = {'key': 'key1', 'value': 'value1'}
-            data = (struct.pack("%ds%ds" % (
+            data = struct.pack(
+                "%ds%ds"
+                % (
                     HyperVKvpReportingHandler.HV_KVP_EXCHANGE_MAX_KEY_SIZE,
-                    HyperVKvpReportingHandler.HV_KVP_EXCHANGE_MAX_VALUE_SIZE),
-                    kvp['key'].encode('utf-8'), kvp['value'].encode('utf-8')))
+                    HyperVKvpReportingHandler.HV_KVP_EXCHANGE_MAX_VALUE_SIZE,
+                ),
+                kvp["key"].encode("utf-8"),
+                kvp["value"].encode("utf-8"),
+            )
             f.write(data)
         cur_time = time.time()
         os.utime(self.tmp_file_path, (cur_time, cur_time))
diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py
index b4767f0c..d15fc60b 100644
--- a/tests/unittests/test_sshutil.py
+++ b/tests/unittests/test_sshutil.py
@@ -374,13 +374,13 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase):
 
         sshd_config = self.tmp_path('sshd_config')
         util.write_file(
-                sshd_config,
-                "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys))
+            sshd_config,
+            "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys)
+        )
 
         (auth_key_fn, auth_key_entries) = ssh_util.extract_authorized_keys(
-                fpw.pw_name, sshd_config)
-        content = ssh_util.update_authorized_keys(
-                auth_key_entries, [])
+            fpw.pw_name, sshd_config)
+        content = ssh_util.update_authorized_keys(auth_key_entries, [])
 
         self.assertEqual("%s/.ssh/authorized_keys" % fpw.pw_dir, auth_key_fn)
         self.assertTrue(VALID_CONTENT['rsa'] in content)
@@ -398,11 +398,13 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase):
 
         sshd_config = self.tmp_path('sshd_config')
         util.write_file(
-                sshd_config,
-                "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys))
+            sshd_config,
+            "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys)
+        )
 
         (auth_key_fn, auth_key_entries) = ssh_util.extract_authorized_keys(
-                fpw.pw_name, sshd_config)
+            fpw.pw_name, sshd_config
+        )
         content = ssh_util.update_authorized_keys(auth_key_entries, [])
 
         self.assertEqual("%s/.ssh/authorized_keys" % fpw.pw_dir, auth_key_fn)
diff --git a/tox.ini b/tox.ini
index e04c7791..ebcebc41 100644
--- a/tox.ini
+++ b/tox.ini
@@ -43,11 +43,10 @@ basepython = python2.7
 deps = -r{toxinidir}/test-requirements.txt
 
 [flake8]
-# E126: continuation line over-indented for hanging indent
 # E226: missing whitespace around arithmetic operator
 # W503: line break before binary operator
 # W504: line break after binary operator
-ignore=E126,E226,W503,W504
+ignore=E226,W503,W504
 exclude = .venv,.tox,dist,doc,*egg,.git,build,tools
 per-file-ignores =
     cloudinit/cmd/main.py:E402
-- 
cgit v1.2.3


From 9a97a3f24e196401a9c54e9c7977ef6a03c98aeb Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Tue, 23 Jun 2020 10:08:16 -0400
Subject: distros.networking: initial implementation of layout (#391)

This commit introduces the initial structure for the "cloudinit.net -> cloudinit.distros.networking Hierarchy" refactor, as detailed in [0]. It also updates that section with some changes driven by this initial implementation, as well as adding a lot more specifics to it.

[0] https://cloudinit.readthedocs.io/en/latest/topics/hacking.html#cloudinit-net-cloudinit-distros-networking-hierarchy
---
 HACKING.rst                     | 227 ++++++++++++++++++++++++++++++++++++----
 cloudinit/distros/__init__.py   |   3 +
 cloudinit/distros/bsd.py        |   2 +
 cloudinit/distros/networking.py | 137 ++++++++++++++++++++++++
 4 files changed, 346 insertions(+), 23 deletions(-)
 create mode 100644 cloudinit/distros/networking.py

(limited to 'cloudinit/distros')

diff --git a/HACKING.rst b/HACKING.rst
index 47ac6a86..4e5f56d7 100644
--- a/HACKING.rst
+++ b/HACKING.rst
@@ -347,7 +347,7 @@ intended as documentation for developers involved in the refactoring,
 but also for other developers who may interact with the code being
 refactored in the meantime.
 
-``cloudinit.net`` -> ``cloudinit.distros.Networking`` Hierarchy
+``cloudinit.net`` -> ``cloudinit.distros.networking`` Hierarchy
 ---------------------------------------------------------------
 
 ``cloudinit.net`` was imported from the curtin codebase as a chunk, and
@@ -367,12 +367,13 @@ there may already be differences in tooling that we currently
 work around in less obvious ways.
 
 The high-level plan is to introduce a hierarchy of networking classes
-in ``cloudinit.distros``, which each ``Distro`` subclass will
-reference.  These will capture the differences between networking on
-our various distros, while still allowing easy reuse of code between
+in ``cloudinit.distros.networking``, which each ``Distro`` subclass
+will reference.  These will capture the differences between networking
+on our various distros, while still allowing easy reuse of code between
 distros that share functionality (e.g. most of the Linux networking
-behaviour).  Callers will call ``distro.net.func`` instead of
-``cloudinit.net.func``, which will necessitate access to an
+behaviour).  ``Distro`` objects will instantiate the networking classes
+at ``self.net``, so callers will call ``distro.net.<func>`` instead of
+``cloudinit.net.<func>``; this will necessitate access to an
 instantiated ``Distro`` object.
 
 An implementation note: there may be external consumers of the
@@ -386,9 +387,9 @@ existing API exactly.)
 In more detail:
 
 * The root of this hierarchy will be the
-  ``cloudinit.distros.Networking`` class.  This class will have
-  a corresponding method for every ``cloudinit.net`` function that we
-  identify to be involved in refactoring.  Initially, these methods'
+  ``cloudinit.distros.networking.Networking`` class.  This class will
+  have a corresponding method for every ``cloudinit.net`` function that
+  we identify to be involved in refactoring.  Initially, these methods'
   implementations will simply call the corresponding ``cloudinit.net``
   function.  (This gives us the complete API from day one, for existing
   consumers.)
@@ -413,31 +414,208 @@ In more detail:
   its ``net`` attribute.  (This is the entry point for existing
   consumers to migrate to.)
 * Callers of refactored functions will change from calling
-  ``cloudinit.net.some_func`` to ``distro.net.some_func``, where
-  ``distro`` is an instance of the appropriate ``Distro`` class for
-  this system.  (This will require making such an instance available to
-  callers, which will constitute a large part of the work in this
-  project.)
+  ``cloudinit.net.<func>`` to ``distro.net.<func>``, where ``distro``
+  is an instance of the appropriate ``Distro`` class for this system.
+  (This will require making such an instance available to callers,
+  which will constitute a large part of the work in this project.)
 
 After the initial structure is in place, the work in this refactor will
 consist of replacing the ``cloudinit.net.some_func`` call in each
-``cloudinit.distros.Networking`` method with the actual implementation.
-This can be done incrementally, one function at a time:
-
-* pick an unmigrated ``cloudinit.distros.Networking`` method
-* refactor all of its callers to call the ``distro.net`` method on
-  ``Distro`` instead of the ``cloudinit.net`` function. (This is likely
-  to be the most time-consuming step, as it may require plumbing
-  ``Distro`` objects through to places that previously have not
-  consumed them.)
+``cloudinit.distros.networking.Networking`` method with the actual
+implementation.  This can be done incrementally, one function at a
+time:
+
+* pick an unmigrated ``cloudinit.distros.networking.Networking`` method
+* refactor all of its callers to call the ``distro.net.<func>`` method
+  on ``Distro`` instead of the ``cloudinit.net.<func>`` function. (This
+  is likely to be the most time-consuming step, as it may require
+  plumbing ``Distro`` objects through to places that previously have
+  not consumed them.)
 * refactor its implementation from ``cloudinit.net`` into the
   ``Networking`` hierarchy (e.g. if it has an if/else on BSD, this is
   the time to put the implementations in their respective subclasses)
+
+  * if part of the method contains distro-independent logic, then you
+    may need to create new methods to capture this distro-specific
+    logic; we don't want to replicate common logic in different
+    ``Networking`` subclasses
+  * if after the refactor, the method on the root ``Networking`` class
+    no longer has any implementation, it should be converted to an
+    `abstractmethod`_
+
 * ensure that the new implementation has unit tests (either by moving
   existing tests, or by writing new ones)
+* ensure that the new implementation has a docstring
+* add any appropriate type annotations
+
+  * note that we must follow the constraints described in the "Type
+    Annotations" section above, so you may not be able to write
+    complete annotations
+  * we have `type aliases`_ defined in ``cloudinit.distros.networking``
+    which should be used when applicable
+
 * finally, remove it (and any other now-unused functions) from
   cloudinit.net (to avoid having two parallel implementations)
 
+``cloudinit.net`` Functions/Classes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The functions/classes that need refactoring break down into some broad
+categories:
+
+* helpers for accessing ``/sys`` (that should not be on the top-level
+  ``Networking`` class as they are Linux-specific):
+
+  * ``get_sys_class_path``
+  * ``sys_dev_path``
+  * ``read_sys_net``
+  * ``read_sys_net_safe``
+  * ``read_sys_net_int``
+
+* those that directly access ``/sys`` (via helpers) and should (IMO) be
+  included in the API of the ``Networking`` class:
+
+  * ``generate_fallback_config``
+
+    * the ``config_driver`` parameter is used and passed as a boolean,
+      so we can change the default value to ``False`` (instead of
+      ``None``)
+
+  * ``get_ib_interface_hwaddr``
+  * ``get_interface_mac``
+  * ``interface_has_own_mac``
+  * ``is_bond``
+  * ``is_bridge``
+  * ``is_connected``
+  * ``is_physical``
+  * ``is_present``
+  * ``is_renamed``
+  * ``is_up``
+  * ``is_vlan``
+  * ``is_wireless``
+  * ``wait_for_physdevs``
+
+* those that directly access ``/sys`` (via helpers) but may be
+  Linux-specific concepts or names:
+
+  * ``get_master``
+  * ``device_devid``
+  * ``device_driver``
+
+* those that directly use ``ip``:
+
+  * ``_get_current_rename_info``
+
+    * this has non-distro-specific logic so should potentially be
+      refactored to use helpers on ``self`` instead of ``ip`` directly
+      (rather than being wholesale reimplemented in each of
+      ``BSDNetworking`` or ``LinuxNetworking``)
+    * we can also remove the ``check_downable`` argument, it's never
+      specified so is always ``True``
+
+  * ``_rename_interfaces``
+
+    * this has several internal helper functions which use ``ip``
+      directly, and it calls ``_get_current_rename_info``.  That said,
+      there appears to be a lot of non-distro-specific logic that could
+      live in a function on ``Networking``, so this will require some
+      careful refactoring to avoid duplicating that logic in each of
+      ``BSDNetworking`` and ``LinuxNetworking``.
+    * only the ``renames`` and ``current_info`` parameters are ever
+      passed in (and ``current_info`` only by tests), so we can remove
+      the others from the definition
+
+  * ``EphemeralIPv4Network``
+
+    * this is another case where it mixes distro-specific and
+      non-specific functionality.  Specifically, ``__init__``,
+      ``__enter__`` and ``__exit__`` are non-specific, and the
+      remaining methods are distro-specific.
+    * when refactoring this, the need to track ``cleanup_cmds`` likely
+      means that the distro-specific behaviour cannot be captured only
+      in the ``Networking`` class.  See `this comment in PR #363`_ for
+      more thoughts.
+
+* those that implicitly use ``/sys`` via their call dependencies:
+
+  * ``master_is_bridge_or_bond``
+
+    * appends to ``get_master`` return value, which is a ``/sys`` path
+
+  * ``extract_physdevs``
+
+    * calls ``device_driver`` and ``device_devid`` in both
+      ``_version_*`` impls
+
+  * ``apply_network_config_names``
+
+    * calls ``extract_physdevs``
+    * there is already a ``Distro.apply_network_config_names`` which in
+      the default implementation calls this function; this and its BSD
+      subclass implementations should be refactored at the same time
+    * the ``strict_present`` and ``strict_busy`` parameters are never
+      passed, nor are they used in the function definition, so they can
+      be removed
+
+  * ``get_interfaces``
+
+    * calls ``device_driver``, ``device_devid`` amongst others
+
+  * ``get_ib_hwaddrs_by_interface``
+
+    * calls ``get_interfaces``
+
+* those that may fall into the above categories, but whose use is only
+  related to netfailover (which relies on a Linux-specific network
+  driver, so is unlikely to be relevant elsewhere without a substantial
+  refactor; these probably only need implementing in
+  ``LinuxNetworking``):
+
+  * ``get_dev_features``
+
+  * ``has_netfail_standby_feature``
+
+    * calls ``get_dev_features``
+
+  * ``is_netfailover``
+  * ``is_netfail_master``
+
+    * this is called from ``generate_fallback_config``
+
+  * ``is_netfail_primary``
+  * ``is_netfail_standby``
+
+  * N.B. all of these take an optional ``driver`` argument which is
+    used to pass around a value to avoid having to look it up by
+    calling ``device_driver`` every time.  This is something of a leaky
+    abstraction, and is better served by caching on ``device_driver``
+    or storing the cached value on ``self``, so we can drop the
+    parameter from the new API.
+
+* those that use ``/sys`` (via helpers) and have non-exhaustive BSD
+  logic:
+
+  * ``get_devicelist``
+
+* those that already have separate Linux/BSD implementations:
+
+  * ``find_fallback_nic``
+  * ``get_interfaces_by_mac``
+
+* those that have no OS-specific functionality (so do not need to be
+  refactored):
+
+  * ``ParserError``
+  * ``RendererNotFoundError``
+  * ``has_url_connectivity``
+  * ``is_ip_address``
+  * ``is_ipv4_address``
+  * ``natural_sort_key``
+
+Note that the functions in ``cloudinit.net`` use inconsistent parameter
+names for "string that contains a device name"; we can standardise on
+``devname`` (the most common one) in the refactor.
+
 References
 ~~~~~~~~~~
 
@@ -450,3 +628,6 @@ References
 .. _Mina Galić's email the the cloud-init ML in 2018: https://lists.launchpad.net/cloud-init/msg00185.html
 .. _Mina Galić's email to the cloud-init ML in 2019: https://lists.launchpad.net/cloud-init/msg00237.html
 .. _PR #363: https://github.com/canonical/cloud-init/pull/363
+.. _this comment in PR #363: https://github.com/canonical/cloud-init/pull/363#issuecomment-628829489
+.. _abstractmethod: https://docs.python.org/3/library/abc.html#abc.abstractmethod
+.. _type aliases: https://docs.python.org/3/library/typing.html#type-aliases
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 016ba64d..89940cf0 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -29,6 +29,7 @@ from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.distros.parsers import hosts
+from .networking import LinuxNetworking
 
 
 # Used when a cloud-config module can be run on all cloud-init distibutions.
@@ -67,11 +68,13 @@ class Distro(metaclass=abc.ABCMeta):
     init_cmd = ['service']  # systemctl, service etc
     renderer_configs = {}
     _preferred_ntp_clients = None
+    networking_cls = LinuxNetworking
 
     def __init__(self, name, cfg, paths):
         self._paths = paths
         self._cfg = cfg
         self.name = name
+        self.networking = self.networking_cls()
 
     @abc.abstractmethod
     def install_packages(self, pkglist):
diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py
index c2d1f77d..2ed7a7d5 100644
--- a/cloudinit/distros/bsd.py
+++ b/cloudinit/distros/bsd.py
@@ -7,11 +7,13 @@ from cloudinit import log as logging
 from cloudinit import net
 from cloudinit import subp
 from cloudinit import util
+from .networking import BSDNetworking
 
 LOG = logging.getLogger(__name__)
 
 
 class BSD(distros.Distro):
+    networking_cls = BSDNetworking
     hostname_conf_fn = '/etc/rc.conf'
     rc_conf_fn = "/etc/rc.conf"
 
diff --git a/cloudinit/distros/networking.py b/cloudinit/distros/networking.py
new file mode 100644
index 00000000..f9842889
--- /dev/null
+++ b/cloudinit/distros/networking.py
@@ -0,0 +1,137 @@
+import abc
+
+from cloudinit import net
+
+
+# Type aliases (https://docs.python.org/3/library/typing.html#type-aliases),
+# used to make the signatures of methods a little clearer
+DeviceName = str
+NetworkConfig = dict
+
+
+class Networking(metaclass=abc.ABCMeta):
+    """The root of the Networking hierarchy in cloud-init.
+
+    This is part of an ongoing refactor in the cloud-init codebase, for more
+    details see "``cloudinit.net`` -> ``cloudinit.distros.networking``
+    Hierarchy" in HACKING.rst for full details.
+    """
+
+    def _get_current_rename_info(self) -> dict:
+        return net._get_current_rename_info()
+
+    def _rename_interfaces(self, renames: list, *, current_info=None) -> None:
+        return net._rename_interfaces(renames, current_info=current_info)
+
+    def apply_network_config_names(self, netcfg: NetworkConfig) -> None:
+        return net.apply_network_config_names(netcfg)
+
+    def device_devid(self, devname: DeviceName):
+        return net.device_devid(devname)
+
+    def device_driver(self, devname: DeviceName):
+        return net.device_driver(devname)
+
+    def extract_physdevs(self, netcfg: NetworkConfig) -> list:
+        return net.extract_physdevs(netcfg)
+
+    def find_fallback_nic(self, *, blacklist_drivers=None):
+        return net.find_fallback_nic(blacklist_drivers=blacklist_drivers)
+
+    def generate_fallback_config(
+        self, *, blacklist_drivers=None, config_driver: bool = False
+    ):
+        return net.generate_fallback_config(
+            blacklist_drivers=blacklist_drivers, config_driver=config_driver
+        )
+
+    def get_devicelist(self) -> list:
+        return net.get_devicelist()
+
+    def get_ib_hwaddrs_by_interface(self) -> dict:
+        return net.get_ib_hwaddrs_by_interface()
+
+    def get_ib_interface_hwaddr(
+        self, devname: DeviceName, ethernet_format: bool
+    ):
+        return net.get_ib_interface_hwaddr(devname, ethernet_format)
+
+    def get_interface_mac(self, devname: DeviceName):
+        return net.get_interface_mac(devname)
+
+    def get_interfaces(self) -> list:
+        return net.get_interfaces()
+
+    def get_interfaces_by_mac(self) -> dict:
+        return net.get_interfaces_by_mac()
+
+    def get_master(self, devname: DeviceName):
+        return net.get_master(devname)
+
+    def interface_has_own_mac(
+        self, devname: DeviceName, *, strict: bool = False
+    ) -> bool:
+        return net.interface_has_own_mac(devname, strict=strict)
+
+    def is_bond(self, devname: DeviceName) -> bool:
+        return net.is_bond(devname)
+
+    def is_bridge(self, devname: DeviceName) -> bool:
+        return net.is_bridge(devname)
+
+    def is_connected(self, devname: DeviceName) -> bool:
+        return net.is_connected(devname)
+
+    def is_physical(self, devname: DeviceName) -> bool:
+        return net.is_physical(devname)
+
+    def is_present(self, devname: DeviceName) -> bool:
+        return net.is_present(devname)
+
+    def is_renamed(self, devname: DeviceName) -> bool:
+        return net.is_renamed(devname)
+
+    def is_up(self, devname: DeviceName) -> bool:
+        return net.is_up(devname)
+
+    def is_vlan(self, devname: DeviceName) -> bool:
+        return net.is_vlan(devname)
+
+    def is_wireless(self, devname: DeviceName) -> bool:
+        return net.is_wireless(devname)
+
+    def master_is_bridge_or_bond(self, devname: DeviceName) -> bool:
+        return net.master_is_bridge_or_bond(devname)
+
+    def wait_for_physdevs(
+        self, netcfg: NetworkConfig, *, strict: bool = True
+    ) -> None:
+        return net.wait_for_physdevs(netcfg, strict=strict)
+
+
+class BSDNetworking(Networking):
+    """Implementation of networking functionality shared across BSDs."""
+
+    pass
+
+
+class LinuxNetworking(Networking):
+    """Implementation of networking functionality common to Linux distros."""
+
+    def get_dev_features(self, devname: DeviceName) -> str:
+        return net.get_dev_features(devname)
+
+    def has_netfail_standby_feature(self, devname: DeviceName) -> bool:
+        return net.has_netfail_standby_feature(devname)
+
+    def is_netfailover(self, devname: DeviceName) -> bool:
+        return net.is_netfailover(devname)
+
+    def is_netfail_master(self, devname: DeviceName) -> bool:
+        return net.is_netfail_master(devname)
+
+    def is_netfail_primary(self, devname: DeviceName) -> bool:
+        return net.is_netfail_primary(devname)
+
+    def is_netfail_standby(self, devname: DeviceName) -> bool:
+        return net.is_netfail_standby(devname)
-- 
cgit v1.2.3


From 31b540524fd1dd5ae79bc703f581339fff1883e9 Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Wed, 24 Jun 2020 15:22:45 -0400
Subject: net/networking: remove unused functions/methods (#453)

Namely, is_connected, is_wireless and is_present.  None of these are
used in the cloud-init codebase, so remove the dead code (instead of
refactoring it).
---
 HACKING.rst                      |  3 ---
 cloudinit/distros/networking.py  |  9 ---------
 cloudinit/net/__init__.py        | 22 ----------------------
 cloudinit/net/tests/test_init.py | 26 --------------------------
 4 files changed, 60 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/HACKING.rst b/HACKING.rst
index 4e5f56d7..d98e1d40 100644
--- a/HACKING.rst
+++ b/HACKING.rst
@@ -486,13 +486,10 @@ categories:
   * ``interface_has_own_mac``
   * ``is_bond``
   * ``is_bridge``
-  * ``is_connected``
   * ``is_physical``
-  * ``is_present``
   * ``is_renamed``
   * ``is_up``
   * ``is_vlan``
-  * ``is_wireless``
   * ``wait_for_physdevs``
 
 * those that directly access ``/sys`` (via helpers) but may be
diff --git a/cloudinit/distros/networking.py b/cloudinit/distros/networking.py
index f9842889..eecdccc6 100644
--- a/cloudinit/distros/networking.py
+++ b/cloudinit/distros/networking.py
@@ -79,15 +79,9 @@ class Networking(metaclass=abc.ABCMeta):
     def is_bridge(self, devname: DeviceName) -> bool:
         return net.is_bridge(devname)
 
-    def is_connected(self, devname: DeviceName) -> bool:
-        return net.is_connected(devname)
-
     def is_physical(self, devname: DeviceName) -> bool:
         return net.is_physical(devname)
 
-    def is_present(self, devname: DeviceName) -> bool:
-        return net.is_present(devname)
-
     def is_renamed(self, devname: DeviceName) -> bool:
         return net.is_renamed(devname)
 
@@ -97,9 +91,6 @@ class Networking(metaclass=abc.ABCMeta):
     def is_vlan(self, devname: DeviceName) -> bool:
         return net.is_vlan(devname)
 
-    def is_wireless(self, devname: DeviceName) -> bool:
-        return net.is_wireless(devname)
-
     def master_is_bridge_or_bond(self, devname: DeviceName) -> bool:
         return net.master_is_bridge_or_bond(devname)
 
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index b40cb154..fd5a1b3e 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -99,10 +99,6 @@ def is_up(devname):
     return read_sys_net_safe(devname, "operstate", translate=translate)
 
 
-def is_wireless(devname):
-    return os.path.exists(sys_dev_path(devname, "wireless"))
-
-
 def is_bridge(devname):
     return os.path.exists(sys_dev_path(devname, "bridge"))
 
@@ -266,28 +262,10 @@ def is_vlan(devname):
     return 'DEVTYPE=vlan' in uevent.splitlines()
 
 
-def is_connected(devname):
-    # is_connected isn't really as simple as that.  2 is
-    # 'physically connected'. 3 is 'not connected'. but a wlan interface will
-    # always show 3.
-    iflink = read_sys_net_safe(devname, "iflink")
-    if iflink == "2":
-        return True
-    if not is_wireless(devname):
-        return False
-    LOG.debug("'%s' is wireless, basing 'connected' on carrier", devname)
-    return read_sys_net_safe(devname, "carrier",
-                             translate={'0': False, '1': True})
-
-
 def is_physical(devname):
     return os.path.exists(sys_dev_path(devname, "device"))
 
 
-def is_present(devname):
-    return os.path.exists(sys_dev_path(devname))
-
-
 def device_driver(devname):
     """Return the device driver for net device named 'devname'."""
     driver = None
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index 1c0a16a8..e36d4387 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -143,12 +143,6 @@ class TestReadSysNet(CiTestCase):
             write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), state)
             self.assertFalse(net.is_up('eth0'))
 
-    def test_is_wireless(self):
-        """is_wireless is True when /sys/net/devname/wireless exists."""
-        self.assertFalse(net.is_wireless('eth0'))
-        ensure_file(os.path.join(self.sysdir, 'eth0', 'wireless'))
-        self.assertTrue(net.is_wireless('eth0'))
-
     def test_is_bridge(self):
         """is_bridge is True when /sys/net/devname/bridge exists."""
         self.assertFalse(net.is_bridge('eth0'))
@@ -204,32 +198,12 @@ class TestReadSysNet(CiTestCase):
         write_file(os.path.join(self.sysdir, 'eth0', 'uevent'), content)
         self.assertTrue(net.is_vlan('eth0'))
 
-    def test_is_connected_when_physically_connected(self):
-        """is_connected is True when /sys/net/devname/iflink reports 2."""
-        self.assertFalse(net.is_connected('eth0'))
-        write_file(os.path.join(self.sysdir, 'eth0', 'iflink'), "2")
-        self.assertTrue(net.is_connected('eth0'))
-
-    def test_is_connected_when_wireless_and_carrier_active(self):
-        """is_connected is True if wireless /sys/net/devname/carrier is 1."""
-        self.assertFalse(net.is_connected('eth0'))
-        ensure_file(os.path.join(self.sysdir, 'eth0', 'wireless'))
-        self.assertFalse(net.is_connected('eth0'))
-        write_file(os.path.join(self.sysdir, 'eth0', 'carrier'), "1")
-        self.assertTrue(net.is_connected('eth0'))
-
     def test_is_physical(self):
         """is_physical is True when /sys/net/devname/device exists."""
         self.assertFalse(net.is_physical('eth0'))
         ensure_file(os.path.join(self.sysdir, 'eth0', 'device'))
         self.assertTrue(net.is_physical('eth0'))
 
-    def test_is_present(self):
-        """is_present is True when /sys/net/devname exists."""
-        self.assertFalse(net.is_present('eth0'))
-        ensure_file(os.path.join(self.sysdir, 'eth0', 'device'))
-        self.assertTrue(net.is_present('eth0'))
-
 
 class TestGenerateFallbackConfig(CiTestCase):
 
-- 
cgit v1.2.3


From 882f1a5f2d5bafd08e6900a2782c3affa67c9d86 Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Tue, 30 Jun 2020 14:19:38 -0400
Subject: networking: refactor is_physical from cloudinit.net (#457)

As the first refactor PR, this also includes the initial structure for tests.

LP: #1884619
---
 cloudinit/distros/networking.py                    |  16 ++-
 cloudinit/distros/tests/test_networking.py         |  42 ++++++
 cloudinit/net/__init__.py                          |   4 -
 cloudinit/net/tests/test_init.py                   |   6 -
 cloudinit/sources/DataSourceDigitalOcean.py        |   2 +-
 cloudinit/sources/DataSourceOpenNebula.py          |  30 +++--
 cloudinit/sources/helpers/digitalocean.py          |  12 +-
 tests/unittests/test_datasource/test_opennebula.py | 149 +++++++++++++--------
 8 files changed, 181 insertions(+), 80 deletions(-)
 create mode 100644 cloudinit/distros/tests/test_networking.py

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/networking.py b/cloudinit/distros/networking.py
index eecdccc6..e421a2ce 100644
--- a/cloudinit/distros/networking.py
+++ b/cloudinit/distros/networking.py
@@ -1,4 +1,5 @@
 import abc
+import os
 
 from cloudinit import net
 
@@ -79,8 +80,15 @@ class Networking(metaclass=abc.ABCMeta):
     def is_bridge(self, devname: DeviceName) -> bool:
         return net.is_bridge(devname)
 
+    @abc.abstractmethod
     def is_physical(self, devname: DeviceName) -> bool:
-        return net.is_physical(devname)
+        """
+        Is ``devname`` a physical network device?
+
+        Examples of non-physical network devices: bonds, bridges, tunnels,
+        loopback devices.
+        """
+        pass
 
     def is_renamed(self, devname: DeviceName) -> bool:
         return net.is_renamed(devname)
@@ -103,7 +111,8 @@ class Networking(metaclass=abc.ABCMeta):
 class BSDNetworking(Networking):
     """Implementation of networking functionality shared across BSDs."""
 
-    pass
+    def is_physical(self, devname: DeviceName) -> bool:
+        raise NotImplementedError()
 
 
 class LinuxNetworking(Networking):
@@ -126,3 +135,6 @@ class LinuxNetworking(Networking):
 
     def is_netfail_standby(self, devname: DeviceName) -> bool:
         return net.is_netfail_standby(devname)
+
+    def is_physical(self, devname: DeviceName) -> bool:
+        return os.path.exists(net.sys_dev_path(devname, "device"))
diff --git a/cloudinit/distros/tests/test_networking.py b/cloudinit/distros/tests/test_networking.py
new file mode 100644
index 00000000..2acb12f4
--- /dev/null
+++ b/cloudinit/distros/tests/test_networking.py
@@ -0,0 +1,42 @@
+from unittest import mock
+
+import pytest
+
+from cloudinit.distros.networking import BSDNetworking, LinuxNetworking
+
+
+@pytest.yield_fixture
+def sys_class_net(tmpdir):
+    sys_class_net_path = tmpdir.join("sys/class/net")
+    sys_class_net_path.ensure_dir()
+    with mock.patch(
+        "cloudinit.net.get_sys_class_path",
+        return_value=sys_class_net_path.strpath + "/",
+    ):
+        yield sys_class_net_path
+
+
+class TestBSDNetworkingIsPhysical:
+    def test_raises_notimplementederror(self):
+        with pytest.raises(NotImplementedError):
+            BSDNetworking().is_physical("eth0")
+
+
+class TestLinuxNetworkingIsPhysical:
+    def test_returns_false_by_default(self, sys_class_net):
+        assert not LinuxNetworking().is_physical("eth0")
+
+    def test_returns_false_if_devname_exists_but_not_physical(
+        self, sys_class_net
+    ):
+        devname = "eth0"
+        sys_class_net.join(devname).mkdir()
+        assert not LinuxNetworking().is_physical(devname)
+
+    def test_returns_true_if_device_is_physical(self, sys_class_net):
+        devname = "eth0"
+        device_dir = sys_class_net.join(devname)
+        device_dir.mkdir()
+        device_dir.join("device").write("")
+
+        assert LinuxNetworking().is_physical(devname)
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index fd5a1b3e..9d8c7ba9 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -262,10 +262,6 @@ def is_vlan(devname):
     return 'DEVTYPE=vlan' in uevent.splitlines()
 
 
-def is_physical(devname):
-    return os.path.exists(sys_dev_path(devname, "device"))
-
-
 def device_driver(devname):
     """Return the device driver for net device named 'devname'."""
     driver = None
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index e36d4387..eb458c39 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -198,12 +198,6 @@ class TestReadSysNet(CiTestCase):
         write_file(os.path.join(self.sysdir, 'eth0', 'uevent'), content)
         self.assertTrue(net.is_vlan('eth0'))
 
-    def test_is_physical(self):
-        """is_physical is True when /sys/net/devname/device exists."""
-        self.assertFalse(net.is_physical('eth0'))
-        ensure_file(os.path.join(self.sysdir, 'eth0', 'device'))
-        self.assertTrue(net.is_physical('eth0'))
-
 
 class TestGenerateFallbackConfig(CiTestCase):
 
diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py
index e0ef665e..5040ce5b 100644
--- a/cloudinit/sources/DataSourceDigitalOcean.py
+++ b/cloudinit/sources/DataSourceDigitalOcean.py
@@ -58,7 +58,7 @@ class DataSourceDigitalOcean(sources.DataSource):
 
         ipv4LL_nic = None
         if self.use_ip4LL:
-            ipv4LL_nic = do_helper.assign_ipv4_link_local()
+            ipv4LL_nic = do_helper.assign_ipv4_link_local(self.distro)
 
         md = do_helper.read_metadata(
             self.metadata_address, timeout=self.timeout,
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index c7fdc2a5..12b1f94f 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -13,6 +13,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import collections
+import functools
 import os
 import pwd
 import re
@@ -60,10 +61,19 @@ class DataSourceOpenNebula(sources.DataSource):
         for cdev in candidates:
             try:
                 if os.path.isdir(self.seed_dir):
-                    results = read_context_disk_dir(cdev, asuser=parseuser)
+                    results = read_context_disk_dir(
+                        cdev, self.distro, asuser=parseuser
+                    )
                 elif cdev.startswith("/dev"):
-                    results = util.mount_cb(cdev, read_context_disk_dir,
-                                            data=parseuser)
+                    # util.mount_cb only handles passing a single argument
+                    # through to the wrapped function, so we have to partially
+                    # apply the function to pass in `distro`.  See LP: #1884979
+                    partially_applied_func = functools.partial(
+                        read_context_disk_dir,
+                        asuser=parseuser,
+                        distro=self.distro,
+                    )
+                    results = util.mount_cb(cdev, partially_applied_func)
             except NonContextDiskDir:
                 continue
             except BrokenContextDiskDir as exc:
@@ -129,10 +139,10 @@ class BrokenContextDiskDir(Exception):
 
 
 class OpenNebulaNetwork(object):
-    def __init__(self, context, system_nics_by_mac=None):
+    def __init__(self, context, distro, system_nics_by_mac=None):
         self.context = context
         if system_nics_by_mac is None:
-            system_nics_by_mac = get_physical_nics_by_mac()
+            system_nics_by_mac = get_physical_nics_by_mac(distro)
         self.ifaces = collections.OrderedDict(
             [k for k in sorted(system_nics_by_mac.items(),
                                key=lambda k: net.natural_sort_key(k[1]))])
@@ -367,7 +377,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None,
     return ret
 
 
-def read_context_disk_dir(source_dir, asuser=None):
+def read_context_disk_dir(source_dir, distro, asuser=None):
     """
     read_context_disk_dir(source_dir):
     read source_dir and return a tuple with metadata dict and user-data
@@ -450,15 +460,17 @@ def read_context_disk_dir(source_dir, asuser=None):
     # http://docs.opennebula.org/5.4/operation/references/template.html#context-section
     ipaddr_keys = [k for k in context if re.match(r'^ETH\d+_IP.*$', k)]
     if ipaddr_keys:
-        onet = OpenNebulaNetwork(context)
+        onet = OpenNebulaNetwork(context, distro)
         results['network-interfaces'] = onet.gen_conf()
 
     return results
 
 
-def get_physical_nics_by_mac():
+def get_physical_nics_by_mac(distro):
     devs = net.get_interfaces_by_mac()
-    return dict([(m, n) for m, n in devs.items() if net.is_physical(n)])
+    return dict(
+        [(m, n) for m, n in devs.items() if distro.networking.is_physical(n)]
+    )
 
 
 # Legacy: Must be present in case we load an old pkl object
diff --git a/cloudinit/sources/helpers/digitalocean.py b/cloudinit/sources/helpers/digitalocean.py
index f5bbe46a..b545c4d6 100644
--- a/cloudinit/sources/helpers/digitalocean.py
+++ b/cloudinit/sources/helpers/digitalocean.py
@@ -16,7 +16,7 @@ NIC_MAP = {'public': 'eth0', 'private': 'eth1'}
 LOG = logging.getLogger(__name__)
 
 
-def assign_ipv4_link_local(nic=None):
+def assign_ipv4_link_local(distro, nic=None):
     """Bring up NIC using an address using link-local (ip4LL) IPs. On
        DigitalOcean, the link-local domain is per-droplet routed, so there
        is no risk of collisions. However, to be more safe, the ip4LL
@@ -24,7 +24,7 @@ def assign_ipv4_link_local(nic=None):
     """
 
     if not nic:
-        nic = get_link_local_nic()
+        nic = get_link_local_nic(distro)
         LOG.debug("selected interface '%s' for reading metadata", nic)
 
     if not nic:
@@ -54,8 +54,12 @@ def assign_ipv4_link_local(nic=None):
     return nic
 
 
-def get_link_local_nic():
-    nics = [f for f in cloudnet.get_devicelist() if cloudnet.is_physical(f)]
+def get_link_local_nic(distro):
+    nics = [
+        f
+        for f in cloudnet.get_devicelist()
+        if distro.networking.is_physical(f)
+    ]
     if not nics:
         return None
     return min(nics, key=lambda d: cloudnet.read_sys_net_int(d, 'ifindex'))
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index 7c859c8a..80841182 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -132,18 +132,18 @@ class TestOpenNebulaDataSource(CiTestCase):
 
     def test_seed_dir_non_contextdisk(self):
         self.assertRaises(ds.NonContextDiskDir, ds.read_context_disk_dir,
-                          self.seed_dir)
+                          self.seed_dir, mock.Mock())
 
     def test_seed_dir_empty1_context(self):
         populate_dir(self.seed_dir, {'context.sh': ''})
-        results = ds.read_context_disk_dir(self.seed_dir)
+        results = ds.read_context_disk_dir(self.seed_dir, mock.Mock())
 
         self.assertIsNone(results['userdata'])
         self.assertEqual(results['metadata'], {})
 
     def test_seed_dir_empty2_context(self):
         populate_context_dir(self.seed_dir, {})
-        results = ds.read_context_disk_dir(self.seed_dir)
+        results = ds.read_context_disk_dir(self.seed_dir, mock.Mock())
 
         self.assertIsNone(results['userdata'])
         self.assertEqual(results['metadata'], {})
@@ -153,11 +153,11 @@ class TestOpenNebulaDataSource(CiTestCase):
 
         self.assertRaises(ds.BrokenContextDiskDir,
                           ds.read_context_disk_dir,
-                          self.seed_dir)
+                          self.seed_dir, mock.Mock())
 
     def test_context_parser(self):
         populate_context_dir(self.seed_dir, TEST_VARS)
-        results = ds.read_context_disk_dir(self.seed_dir)
+        results = ds.read_context_disk_dir(self.seed_dir, mock.Mock())
 
         self.assertTrue('metadata' in results)
         self.assertEqual(TEST_VARS, results['metadata'])
@@ -168,7 +168,7 @@ class TestOpenNebulaDataSource(CiTestCase):
             for k in ('SSH_KEY', 'SSH_PUBLIC_KEY'):
                 my_d = os.path.join(self.tmp, "%s-%i" % (k, c))
                 populate_context_dir(my_d, {k: '\n'.join(public_keys)})
-                results = ds.read_context_disk_dir(my_d)
+                results = ds.read_context_disk_dir(my_d, mock.Mock())
 
                 self.assertTrue('metadata' in results)
                 self.assertTrue('public-keys' in results['metadata'])
@@ -182,7 +182,7 @@ class TestOpenNebulaDataSource(CiTestCase):
             my_d = os.path.join(self.tmp, k)
             populate_context_dir(my_d, {k: USER_DATA,
                                         'USERDATA_ENCODING': ''})
-            results = ds.read_context_disk_dir(my_d)
+            results = ds.read_context_disk_dir(my_d, mock.Mock())
 
             self.assertTrue('userdata' in results)
             self.assertEqual(USER_DATA, results['userdata'])
@@ -192,7 +192,7 @@ class TestOpenNebulaDataSource(CiTestCase):
         for k in ('USER_DATA', 'USERDATA'):
             my_d = os.path.join(self.tmp, k)
             populate_context_dir(my_d, {k: b64userdata})
-            results = ds.read_context_disk_dir(my_d)
+            results = ds.read_context_disk_dir(my_d, mock.Mock())
 
             self.assertTrue('userdata' in results)
             self.assertEqual(b64userdata, results['userdata'])
@@ -202,7 +202,7 @@ class TestOpenNebulaDataSource(CiTestCase):
             my_d = os.path.join(self.tmp, k)
             populate_context_dir(my_d, {k: util.b64e(USER_DATA),
                                         'USERDATA_ENCODING': 'base64'})
-            results = ds.read_context_disk_dir(my_d)
+            results = ds.read_context_disk_dir(my_d, mock.Mock())
 
             self.assertTrue('userdata' in results)
             self.assertEqual(USER_DATA, results['userdata'])
@@ -214,7 +214,7 @@ class TestOpenNebulaDataSource(CiTestCase):
             for k in ('HOSTNAME', 'PUBLIC_IP', 'IP_PUBLIC', 'ETH0_IP'):
                 my_d = os.path.join(self.tmp, k)
                 populate_context_dir(my_d, {k: PUBLIC_IP})
-                results = ds.read_context_disk_dir(my_d)
+                results = ds.read_context_disk_dir(my_d, mock.Mock())
 
                 self.assertTrue('metadata' in results)
                 self.assertTrue('local-hostname' in results['metadata'])
@@ -229,7 +229,7 @@ class TestOpenNebulaDataSource(CiTestCase):
             # without ETH0_MAC
             # for Older OpenNebula?
             populate_context_dir(self.seed_dir, {'ETH0_IP': IP_BY_MACADDR})
-            results = ds.read_context_disk_dir(self.seed_dir)
+            results = ds.read_context_disk_dir(self.seed_dir, mock.Mock())
 
             self.assertTrue('network-interfaces' in results)
             self.assertTrue(
@@ -239,7 +239,7 @@ class TestOpenNebulaDataSource(CiTestCase):
             # ETH0_IP and ETH0_MAC
             populate_context_dir(
                 self.seed_dir, {'ETH0_IP': IP_BY_MACADDR, 'ETH0_MAC': MACADDR})
-            results = ds.read_context_disk_dir(self.seed_dir)
+            results = ds.read_context_disk_dir(self.seed_dir, mock.Mock())
 
             self.assertTrue('network-interfaces' in results)
             self.assertTrue(
@@ -251,7 +251,7 @@ class TestOpenNebulaDataSource(CiTestCase):
             # "AR = [ TYPE = ETHER ]"
             populate_context_dir(
                 self.seed_dir, {'ETH0_IP': '', 'ETH0_MAC': MACADDR})
-            results = ds.read_context_disk_dir(self.seed_dir)
+            results = ds.read_context_disk_dir(self.seed_dir, mock.Mock())
 
             self.assertTrue('network-interfaces' in results)
             self.assertTrue(
@@ -265,7 +265,7 @@ class TestOpenNebulaDataSource(CiTestCase):
                     'ETH0_MAC': MACADDR,
                     'ETH0_MASK': '255.255.0.0'
                 })
-            results = ds.read_context_disk_dir(self.seed_dir)
+            results = ds.read_context_disk_dir(self.seed_dir, mock.Mock())
 
             self.assertTrue('network-interfaces' in results)
             self.assertTrue(
@@ -279,7 +279,7 @@ class TestOpenNebulaDataSource(CiTestCase):
                     'ETH0_MAC': MACADDR,
                     'ETH0_MASK': ''
                 })
-            results = ds.read_context_disk_dir(self.seed_dir)
+            results = ds.read_context_disk_dir(self.seed_dir, mock.Mock())
 
             self.assertTrue('network-interfaces' in results)
             self.assertTrue(
@@ -292,7 +292,7 @@ class TestOpenNebulaDataSource(CiTestCase):
                     'ETH0_IP6': IP6_GLOBAL,
                     'ETH0_MAC': MACADDR,
                 })
-            results = ds.read_context_disk_dir(self.seed_dir)
+            results = ds.read_context_disk_dir(self.seed_dir, mock.Mock())
 
             self.assertTrue('network-interfaces' in results)
             self.assertTrue(
@@ -305,7 +305,7 @@ class TestOpenNebulaDataSource(CiTestCase):
                     'ETH0_IP6_ULA': IP6_ULA,
                     'ETH0_MAC': MACADDR,
                 })
-            results = ds.read_context_disk_dir(self.seed_dir)
+            results = ds.read_context_disk_dir(self.seed_dir, mock.Mock())
 
             self.assertTrue('network-interfaces' in results)
             self.assertTrue(
@@ -319,7 +319,7 @@ class TestOpenNebulaDataSource(CiTestCase):
                     'ETH0_IP6_PREFIX_LENGTH': IP6_PREFIX,
                     'ETH0_MAC': MACADDR,
                 })
-            results = ds.read_context_disk_dir(self.seed_dir)
+            results = ds.read_context_disk_dir(self.seed_dir, mock.Mock())
 
             self.assertTrue('network-interfaces' in results)
             self.assertTrue(
@@ -333,7 +333,7 @@ class TestOpenNebulaDataSource(CiTestCase):
                     'ETH0_IP6_PREFIX_LENGTH': '',
                     'ETH0_MAC': MACADDR,
                 })
-            results = ds.read_context_disk_dir(self.seed_dir)
+            results = ds.read_context_disk_dir(self.seed_dir, mock.Mock())
 
             self.assertTrue('network-interfaces' in results)
             self.assertTrue(
@@ -370,7 +370,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         expected = {
             '02:00:0a:12:01:01': 'ETH0',
             '02:00:0a:12:0f:0f': 'ETH1', }
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         self.assertEqual(expected, net.context_devname)
 
     def test_get_nameservers(self):
@@ -385,21 +385,21 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         expected = {
             'addresses': ['1.2.3.6', '1.2.3.7', '1.2.3.8'],
             'search': ['example.com', 'example.org']}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_nameservers('eth0')
         self.assertEqual(expected, val)
 
     def test_get_mtu(self):
         """Verify get_mtu('device') correctly returns MTU size."""
         context = {'ETH0_MTU': '1280'}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_mtu('eth0')
         self.assertEqual('1280', val)
 
     def test_get_ip(self):
         """Verify get_ip('device') correctly returns IPv4 address."""
         context = {'ETH0_IP': PUBLIC_IP}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_ip('eth0', MACADDR)
         self.assertEqual(PUBLIC_IP, val)
 
@@ -410,7 +410,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         string.
         """
         context = {'ETH0_IP': ''}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_ip('eth0', MACADDR)
         self.assertEqual(IP_BY_MACADDR, val)
 
@@ -423,7 +423,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
             'ETH0_IP6': IP6_GLOBAL,
             'ETH0_IP6_ULA': '', }
         expected = [IP6_GLOBAL]
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_ip6('eth0')
         self.assertEqual(expected, val)
 
@@ -436,7 +436,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
             'ETH0_IP6': '',
             'ETH0_IP6_ULA': IP6_ULA, }
         expected = [IP6_ULA]
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_ip6('eth0')
         self.assertEqual(expected, val)
 
@@ -449,7 +449,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
             'ETH0_IP6': IP6_GLOBAL,
             'ETH0_IP6_ULA': IP6_ULA, }
         expected = [IP6_GLOBAL, IP6_ULA]
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_ip6('eth0')
         self.assertEqual(expected, val)
 
@@ -458,7 +458,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         Verify get_ip6_prefix('device') correctly returns IPv6 prefix.
         """
         context = {'ETH0_IP6_PREFIX_LENGTH': IP6_PREFIX}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_ip6_prefix('eth0')
         self.assertEqual(IP6_PREFIX, val)
 
@@ -469,7 +469,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         string.
         """
         context = {'ETH0_IP6_PREFIX_LENGTH': ''}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_ip6_prefix('eth0')
         self.assertEqual('64', val)
 
@@ -479,7 +479,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         address.
         """
         context = {'ETH0_GATEWAY': '1.2.3.5'}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_gateway('eth0')
         self.assertEqual('1.2.3.5', val)
 
@@ -489,7 +489,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         address.
         """
         context = {'ETH0_GATEWAY6': IP6_GW}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_gateway6('eth0')
         self.assertEqual(IP6_GW, val)
 
@@ -498,7 +498,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         Verify get_mask('device') correctly returns IPv4 subnet mask.
         """
         context = {'ETH0_MASK': '255.255.0.0'}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_mask('eth0')
         self.assertEqual('255.255.0.0', val)
 
@@ -508,7 +508,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         It returns default value '255.255.255.0' if ETH0_MASK has empty string.
         """
         context = {'ETH0_MASK': ''}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_mask('eth0')
         self.assertEqual('255.255.255.0', val)
 
@@ -517,7 +517,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         Verify get_network('device') correctly returns IPv4 network address.
         """
         context = {'ETH0_NETWORK': '1.2.3.0'}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_network('eth0', MACADDR)
         self.assertEqual('1.2.3.0', val)
 
@@ -528,7 +528,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         empty string.
         """
         context = {'ETH0_NETWORK': ''}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_network('eth0', MACADDR)
         self.assertEqual('10.18.1.0', val)
 
@@ -537,7 +537,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         Verify get_field('device', 'name') returns *context* value.
         """
         context = {'ETH9_DUMMY': 'DUMMY_VALUE'}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_field('eth9', 'dummy')
         self.assertEqual('DUMMY_VALUE', val)
 
@@ -547,7 +547,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         value.
         """
         context = {'ETH9_DUMMY': 'DUMMY_VALUE'}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_field('eth9', 'dummy', 'DEFAULT_VALUE')
         self.assertEqual('DUMMY_VALUE', val)
 
@@ -557,7 +557,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         value if context value is empty string.
         """
         context = {'ETH9_DUMMY': ''}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_field('eth9', 'dummy', 'DEFAULT_VALUE')
         self.assertEqual('DEFAULT_VALUE', val)
 
@@ -567,7 +567,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         empty string.
         """
         context = {'ETH9_DUMMY': ''}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_field('eth9', 'dummy')
         self.assertEqual(None, val)
 
@@ -577,7 +577,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
         None.
         """
         context = {'ETH9_DUMMY': None}
-        net = ds.OpenNebulaNetwork(context)
+        net = ds.OpenNebulaNetwork(context, mock.Mock())
         val = net.get_field('eth9', 'dummy')
         self.assertEqual(None, val)
 
@@ -597,7 +597,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
                         'match': {'macaddress': MACADDR},
                         'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
             m_get_phys_by_mac.return_value = {MACADDR: nic}
-            net = ds.OpenNebulaNetwork(context)
+            net = ds.OpenNebulaNetwork(context, mock.Mock())
             self.assertEqual(net.gen_conf(), expected)
 
         # set ETH0_GATEWAY
@@ -613,7 +613,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
                         'match': {'macaddress': MACADDR},
                         'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
             m_get_phys_by_mac.return_value = {MACADDR: nic}
-            net = ds.OpenNebulaNetwork(context)
+            net = ds.OpenNebulaNetwork(context, mock.Mock())
             self.assertEqual(net.gen_conf(), expected)
 
     @mock.patch(DS_PATH + ".get_physical_nics_by_mac")
@@ -632,7 +632,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
                         'match': {'macaddress': MACADDR},
                         'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
             m_get_phys_by_mac.return_value = {MACADDR: nic}
-            net = ds.OpenNebulaNetwork(context)
+            net = ds.OpenNebulaNetwork(context, mock.Mock())
             self.assertEqual(net.gen_conf(), expected)
 
         # set ETH0_GATEWAY6
@@ -648,7 +648,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
                         'match': {'macaddress': MACADDR},
                         'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
             m_get_phys_by_mac.return_value = {MACADDR: nic}
-            net = ds.OpenNebulaNetwork(context)
+            net = ds.OpenNebulaNetwork(context, mock.Mock())
             self.assertEqual(net.gen_conf(), expected)
 
     @mock.patch(DS_PATH + ".get_physical_nics_by_mac")
@@ -669,7 +669,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
                         'match': {'macaddress': MACADDR},
                         'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
             m_get_phys_by_mac.return_value = {MACADDR: nic}
-            net = ds.OpenNebulaNetwork(context)
+            net = ds.OpenNebulaNetwork(context, mock.Mock())
             self.assertEqual(net.gen_conf(), expected)
 
         # set ETH0_IP6, ETH0_IP6_ULA, ETH0_IP6_PREFIX_LENGTH
@@ -689,7 +689,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
                             IP6_GLOBAL + '/' + IP6_PREFIX,
                             IP6_ULA + '/' + IP6_PREFIX]}}}
             m_get_phys_by_mac.return_value = {MACADDR: nic}
-            net = ds.OpenNebulaNetwork(context)
+            net = ds.OpenNebulaNetwork(context, mock.Mock())
             self.assertEqual(net.gen_conf(), expected)
 
     @mock.patch(DS_PATH + ".get_physical_nics_by_mac")
@@ -710,7 +710,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
                         'match': {'macaddress': MACADDR},
                         'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
             m_get_phys_by_mac.return_value = {MACADDR: nic}
-            net = ds.OpenNebulaNetwork(context)
+            net = ds.OpenNebulaNetwork(context, mock.Mock())
             self.assertEqual(net.gen_conf(), expected)
 
         # set DNS, ETH0_DNS, ETH0_SEARCH_DOMAIN
@@ -730,7 +730,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
                         'match': {'macaddress': MACADDR},
                         'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
             m_get_phys_by_mac.return_value = {MACADDR: nic}
-            net = ds.OpenNebulaNetwork(context)
+            net = ds.OpenNebulaNetwork(context, mock.Mock())
             self.assertEqual(net.gen_conf(), expected)
 
     @mock.patch(DS_PATH + ".get_physical_nics_by_mac")
@@ -749,7 +749,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
                         'match': {'macaddress': MACADDR},
                         'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
             m_get_phys_by_mac.return_value = {MACADDR: nic}
-            net = ds.OpenNebulaNetwork(context)
+            net = ds.OpenNebulaNetwork(context, mock.Mock())
             self.assertEqual(net.gen_conf(), expected)
 
         # set ETH0_MTU
@@ -765,14 +765,14 @@ class TestOpenNebulaNetwork(unittest.TestCase):
                         'match': {'macaddress': MACADDR},
                         'addresses': [IP_BY_MACADDR + '/' + IP4_PREFIX]}}}
             m_get_phys_by_mac.return_value = {MACADDR: nic}
-            net = ds.OpenNebulaNetwork(context)
+            net = ds.OpenNebulaNetwork(context, mock.Mock())
             self.assertEqual(net.gen_conf(), expected)
 
     @mock.patch(DS_PATH + ".get_physical_nics_by_mac")
     def test_eth0(self, m_get_phys_by_mac):
         for nic in self.system_nics:
             m_get_phys_by_mac.return_value = {MACADDR: nic}
-            net = ds.OpenNebulaNetwork({})
+            net = ds.OpenNebulaNetwork({}, mock.Mock())
             expected = {
                 'version': 2,
                 'ethernets': {
@@ -782,6 +782,14 @@ class TestOpenNebulaNetwork(unittest.TestCase):
 
             self.assertEqual(net.gen_conf(), expected)
 
+    @mock.patch(DS_PATH + ".get_physical_nics_by_mac")
+    def test_distro_passed_through(self, m_get_physical_nics_by_mac):
+        ds.OpenNebulaNetwork({}, mock.sentinel.distro)
+        self.assertEqual(
+            [mock.call(mock.sentinel.distro)],
+            m_get_physical_nics_by_mac.call_args_list,
+        )
+
     def test_eth0_override(self):
         self.maxDiff = None
         context = {
@@ -800,7 +808,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
             'ETH0_SEARCH_DOMAIN': '',
         }
         for nic in self.system_nics:
-            net = ds.OpenNebulaNetwork(context,
+            net = ds.OpenNebulaNetwork(context, mock.Mock(),
                                        system_nics_by_mac={MACADDR: nic})
             expected = {
                 'version': 2,
@@ -832,7 +840,7 @@ class TestOpenNebulaNetwork(unittest.TestCase):
             'ETH0_SEARCH_DOMAIN': 'example.com example.org',
         }
         for nic in self.system_nics:
-            net = ds.OpenNebulaNetwork(context,
+            net = ds.OpenNebulaNetwork(context, mock.Mock(),
                                        system_nics_by_mac={MACADDR: nic})
 
             expected = {
@@ -886,7 +894,10 @@ class TestOpenNebulaNetwork(unittest.TestCase):
             'ETH3_SEARCH_DOMAIN': 'third.example.com third.example.org',
         }
         net = ds.OpenNebulaNetwork(
-            context, system_nics_by_mac={MAC_1: 'enp0s25', MAC_2: 'enp1s2'})
+            context,
+            mock.Mock(),
+            system_nics_by_mac={MAC_1: 'enp0s25', MAC_2: 'enp1s2'}
+        )
 
         expected = {
             'version': 2,
@@ -926,6 +937,36 @@ class TestParseShellConfig:
         assert ret == {"foo": "bar", "xx": "foo"}
 
 
+class TestGetPhysicalNicsByMac:
+    @pytest.mark.parametrize(
+        "interfaces_by_mac,physical_devs,expected_return",
+        [
+            # No interfaces => empty return
+            ({}, [], {}),
+            # Only virtual interface => empty return
+            ({"mac1": "virtual0"}, [], {}),
+            # Only physical interface => it is returned
+            ({"mac2": "physical0"}, ["physical0"], {"mac2": "physical0"}),
+            # Combination of physical and virtual => only physical returned
+            (
+                {"mac3": "physical1", "mac4": "virtual1"},
+                ["physical1"],
+                {"mac3": "physical1"},
+            ),
+        ],
+    )
+    def test(self, interfaces_by_mac, physical_devs, expected_return):
+        distro = mock.Mock()
+        distro.networking.is_physical.side_effect = (
+            lambda devname: devname in physical_devs
+        )
+        with mock.patch(
+            DS_PATH + ".net.get_interfaces_by_mac",
+            return_value=interfaces_by_mac,
+        ):
+            assert expected_return == ds.get_physical_nics_by_mac(distro)
+
+
 def populate_context_dir(path, variables):
     data = "# Context variables generated by OpenNebula\n"
     for k, v in variables.items():
-- 
cgit v1.2.3


From 3fcdacc8995d6908858aceaf1da7ee5ff090fc04 Mon Sep 17 00:00:00 2001
From: lucasmoura <lucas.moura@canonical.com>
Date: Tue, 30 Jun 2020 19:25:26 -0300
Subject: Disable ec2 mirror for non aws instances (#390)

For versions before 20.2, we allowed the use of ec2 mirrors if the datasource availability_zone matches one of the ec2 regions. We are now updating that behavior to allow allow the use of ec2 mirrors on ec2 instances or if the user directly passes an an ec2 mirror url through #cloud-config apt directives.

LP: #1456277
---
 cloudinit/distros/__init__.py                      |  10 +-
 cloudinit/distros/tests/test_init.py               |  35 +++-
 cloudinit/features.py                              |  11 ++
 tests/unittests/test_distros/test_generic.py       | 186 +++++++++++++--------
 .../test_handler_apt_configure_sources_list_v1.py  |   6 +-
 .../test_handler/test_handler_apt_source_v1.py     |   7 +
 .../test_handler/test_handler_apt_source_v3.py     |  27 ++-
 7 files changed, 193 insertions(+), 89 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 89940cf0..2fc91bbc 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -28,6 +28,9 @@ from cloudinit import type_utils
 from cloudinit import subp
 from cloudinit import util
 
+from cloudinit.features import \
+    ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES
+
 from cloudinit.distros.parsers import hosts
 from .networking import LinuxNetworking
 
@@ -849,7 +852,12 @@ def _get_package_mirror_info(mirror_info, data_source=None,
         # ec2 availability zones are named cc-direction-[0-9][a-d] (us-east-1b)
         # the region is us-east-1. so region = az[0:-1]
         if _EC2_AZ_RE.match(data_source.availability_zone):
-            subst['ec2_region'] = "%s" % data_source.availability_zone[0:-1]
+            ec2_region = data_source.availability_zone[0:-1]
+
+            if ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES:
+                subst['ec2_region'] = "%s" % ec2_region
+            elif data_source.platform_type == "ec2":
+                subst['ec2_region'] = "%s" % ec2_region
 
     if data_source and data_source.region:
         subst['region'] = data_source.region
diff --git a/cloudinit/distros/tests/test_init.py b/cloudinit/distros/tests/test_init.py
index 40939133..db534654 100644
--- a/cloudinit/distros/tests/test_init.py
+++ b/cloudinit/distros/tests/test_init.py
@@ -67,6 +67,9 @@ class TestGetPackageMirrorInfo:
         assert {'primary': 'http://other'} == _get_package_mirror_info(
             mirror_info, mirror_filter=lambda x: False)
 
+    @pytest.mark.parametrize('allow_ec2_mirror, platform_type', [
+        (True, 'ec2')
+    ])
     @pytest.mark.parametrize('availability_zone,region,patterns,expected', (
         # Test ec2_region alone
         ('fk-fake-1f', None, ['http://EC2-%(ec2_region)s/ubuntu'],
@@ -120,16 +123,34 @@ class TestGetPackageMirrorInfo:
          ['http://%(region)s/ubuntu'], ['http://fk-fake-1/ubuntu'])
         for invalid_char in INVALID_URL_CHARS
     ))
-    def test_substitution(self, availability_zone, region, patterns, expected):
+    def test_valid_substitution(self,
+                                allow_ec2_mirror,
+                                platform_type,
+                                availability_zone,
+                                region,
+                                patterns,
+                                expected):
         """Test substitution works as expected."""
+        flag_path = "cloudinit.distros." \
+                    "ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES"
+
         m_data_source = mock.Mock(
-            availability_zone=availability_zone, region=region
+            availability_zone=availability_zone,
+            region=region,
+            platform_type=platform_type
         )
         mirror_info = {'search': {'primary': patterns}}
 
-        ret = _get_package_mirror_info(
-            mirror_info,
-            data_source=m_data_source,
-            mirror_filter=lambda x: x
-        )
+        with mock.patch(flag_path, allow_ec2_mirror):
+            ret = _get_package_mirror_info(
+                mirror_info,
+                data_source=m_data_source,
+                mirror_filter=lambda x: x
+            )
+        print(allow_ec2_mirror)
+        print(platform_type)
+        print(availability_zone)
+        print(region)
+        print(patterns)
+        print(expected)
         assert {'primary': expected} == ret
diff --git a/cloudinit/features.py b/cloudinit/features.py
index e455213d..c44fa29e 100644
--- a/cloudinit/features.py
+++ b/cloudinit/features.py
@@ -26,6 +26,17 @@ After the 20.2 release, we instead raise an exception.
 This flag can be removed after Focal is no longer supported
 """
 
+
+ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES = False
+"""
+When configuring apt mirrors, old behavior is to allow
+the use of ec2 mirrors if the datasource availability_zone format
+matches one of the possible aws ec2 regions. After the 20.2 release, we
+no longer publish ec2 region mirror urls on non-AWS cloud platforms.
+Besides feature_overrides.py, users can override this by providing
+#cloud-config apt directives.
+"""
+
 try:
     # pylint: disable=wildcard-import
     from cloudinit.feature_overrides import *  # noqa
diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py
index 6b6c1566..44607489 100644
--- a/tests/unittests/test_distros/test_generic.py
+++ b/tests/unittests/test_distros/test_generic.py
@@ -6,6 +6,7 @@ from cloudinit import util
 from cloudinit.tests import helpers
 
 import os
+import pytest
 import shutil
 import tempfile
 from unittest import mock
@@ -37,24 +38,6 @@ gapmi = distros._get_arch_package_mirror_info
 
 class TestGenericDistro(helpers.FilesystemMockingTestCase):
 
-    def return_first(self, mlist):
-        if not mlist:
-            return None
-        return mlist[0]
-
-    def return_second(self, mlist):
-        if not mlist:
-            return None
-        return mlist[1]
-
-    def return_none(self, _mlist):
-        return None
-
-    def return_last(self, mlist):
-        if not mlist:
-            return None
-        return(mlist[-1])
-
     def setUp(self):
         super(TestGenericDistro, self).setUp()
         # Make a temp directoy for tests to use.
@@ -145,61 +128,6 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase):
         arch_mirrors = gapmi(package_mirrors, arch="amd64")
         self.assertEqual(package_mirrors[0], arch_mirrors)
 
-    def test_get_package_mirror_info_az_ec2(self):
-        arch_mirrors = gapmi(package_mirrors, arch="amd64")
-        data_source_mock = mock.Mock(availability_zone="us-east-1a")
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_first)
-        self.assertEqual(results,
-                         {'primary': 'http://us-east-1.ec2/',
-                          'security': 'http://security-mirror1-intel'})
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_second)
-        self.assertEqual(results,
-                         {'primary': 'http://us-east-1a.clouds/',
-                          'security': 'http://security-mirror2-intel'})
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_none)
-        self.assertEqual(results, package_mirrors[0]['failsafe'])
-
-    def test_get_package_mirror_info_az_non_ec2(self):
-        arch_mirrors = gapmi(package_mirrors, arch="amd64")
-        data_source_mock = mock.Mock(availability_zone="nova.cloudvendor")
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_first)
-        self.assertEqual(results,
-                         {'primary': 'http://nova.cloudvendor.clouds/',
-                          'security': 'http://security-mirror1-intel'})
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_last)
-        self.assertEqual(results,
-                         {'primary': 'http://nova.cloudvendor.clouds/',
-                          'security': 'http://security-mirror2-intel'})
-
-    def test_get_package_mirror_info_none(self):
-        arch_mirrors = gapmi(package_mirrors, arch="amd64")
-        data_source_mock = mock.Mock(availability_zone=None)
-
-        # because both search entries here replacement based on
-        # availability-zone, the filter will be called with an empty list and
-        # failsafe should be taken.
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_first)
-        self.assertEqual(results,
-                         {'primary': 'http://fs-primary-intel',
-                          'security': 'http://security-mirror1-intel'})
-
-        results = gpmi(arch_mirrors, data_source=data_source_mock,
-                       mirror_filter=self.return_last)
-        self.assertEqual(results,
-                         {'primary': 'http://fs-primary-intel',
-                          'security': 'http://security-mirror2-intel'})
-
     def test_systemd_in_use(self):
         cls = distros.fetch("ubuntu")
         d = cls("ubuntu", {}, None)
@@ -259,4 +187,116 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase):
             ["pw", "usermod", "myuser", "-p", "01-Jan-1970"])
 
 
+class TestGetPackageMirrors:
+
+    def return_first(self, mlist):
+        if not mlist:
+            return None
+        return mlist[0]
+
+    def return_second(self, mlist):
+        if not mlist:
+            return None
+
+        return mlist[1] if len(mlist) > 1 else None
+
+    def return_none(self, _mlist):
+        return None
+
+    def return_last(self, mlist):
+        if not mlist:
+            return None
+        return(mlist[-1])
+
+    @pytest.mark.parametrize(
+        "allow_ec2_mirror, platform_type, mirrors",
+        [
+            (True, "ec2", [
+                {'primary': 'http://us-east-1.ec2/',
+                 'security': 'http://security-mirror1-intel'},
+                {'primary': 'http://us-east-1a.clouds/',
+                 'security': 'http://security-mirror2-intel'}
+            ]),
+            (True, "other", [
+                {'primary': 'http://us-east-1.ec2/',
+                 'security': 'http://security-mirror1-intel'},
+                {'primary': 'http://us-east-1a.clouds/',
+                 'security': 'http://security-mirror2-intel'}
+            ]),
+            (False, "ec2", [
+                {'primary': 'http://us-east-1.ec2/',
+                 'security': 'http://security-mirror1-intel'},
+                {'primary': 'http://us-east-1a.clouds/',
+                 'security': 'http://security-mirror2-intel'}
+            ]),
+            (False, "other", [
+                {'primary': 'http://us-east-1a.clouds/',
+                 'security': 'http://security-mirror1-intel'},
+                {'primary': 'http://fs-primary-intel',
+                 'security': 'http://security-mirror2-intel'}
+            ])
+        ])
+    def test_get_package_mirror_info_az_ec2(self,
+                                            allow_ec2_mirror,
+                                            platform_type,
+                                            mirrors):
+        flag_path = "cloudinit.distros." \
+                    "ALLOW_EC2_MIRRORS_ON_NON_AWS_INSTANCE_TYPES"
+        with mock.patch(flag_path, allow_ec2_mirror):
+            arch_mirrors = gapmi(package_mirrors, arch="amd64")
+            data_source_mock = mock.Mock(
+                availability_zone="us-east-1a",
+                platform_type=platform_type)
+
+            results = gpmi(arch_mirrors, data_source=data_source_mock,
+                           mirror_filter=self.return_first)
+            assert(results == mirrors[0])
+
+            results = gpmi(arch_mirrors, data_source=data_source_mock,
+                           mirror_filter=self.return_second)
+            assert(results == mirrors[1])
+
+            results = gpmi(arch_mirrors, data_source=data_source_mock,
+                           mirror_filter=self.return_none)
+            assert(results == package_mirrors[0]['failsafe'])
+
+    def test_get_package_mirror_info_az_non_ec2(self):
+        arch_mirrors = gapmi(package_mirrors, arch="amd64")
+        data_source_mock = mock.Mock(availability_zone="nova.cloudvendor")
+
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
+                       mirror_filter=self.return_first)
+        assert(results == {
+            'primary': 'http://nova.cloudvendor.clouds/',
+            'security': 'http://security-mirror1-intel'}
+        )
+
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
+                       mirror_filter=self.return_last)
+        assert(results == {
+            'primary': 'http://nova.cloudvendor.clouds/',
+            'security': 'http://security-mirror2-intel'}
+        )
+
+    def test_get_package_mirror_info_none(self):
+        arch_mirrors = gapmi(package_mirrors, arch="amd64")
+        data_source_mock = mock.Mock(availability_zone=None)
+
+        # because both search entries here replacement based on
+        # availability-zone, the filter will be called with an empty list and
+        # failsafe should be taken.
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
+                       mirror_filter=self.return_first)
+        assert(results == {
+            'primary': 'http://fs-primary-intel',
+            'security': 'http://security-mirror1-intel'}
+        )
+
+        results = gpmi(arch_mirrors, data_source=data_source_mock,
+                       mirror_filter=self.return_last)
+        assert(results == {
+            'primary': 'http://fs-primary-intel',
+            'security': 'http://security-mirror2-intel'}
+        )
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
index e5382544..369480be 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py
@@ -101,6 +101,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
             cfg = {'apt_mirror_search': mirror}
         else:
             cfg = {'apt_mirror': mirror}
+
         mycloud = self._get_cloud(distro)
 
         with mock.patch.object(util, 'write_file') as mockwf:
@@ -108,8 +109,9 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase):
                                    return_value="faketmpl") as mocklf:
                 with mock.patch.object(os.path, 'isfile',
                                        return_value=True) as mockisfile:
-                    with mock.patch.object(templater, 'render_string',
-                                           return_value="fake") as mockrnd:
+                    with mock.patch.object(
+                        templater, 'render_string',
+                            return_value='fake') as mockrnd:
                         with mock.patch.object(util, 'rename'):
                             cc_apt_configure.handle("test", cfg, mycloud,
                                                     LOG, None)
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py
index f2349157..367971cb 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v1.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py
@@ -43,10 +43,17 @@ class FakeDistro(object):
         return
 
 
+class FakeDatasource:
+    """Fake Datasource helper object"""
+    def __init__(self):
+        self.region = 'region'
+
+
 class FakeCloud(object):
     """Fake Cloud helper object"""
     def __init__(self):
         self.distro = FakeDistro()
+        self.datasource = FakeDatasource()
 
 
 class TestAptSourceConfig(TestCase):
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py
index 220100e2..ac847238 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py
@@ -49,6 +49,18 @@ MOCK_LSB_RELEASE_DATA = {
     'release': '18.04', 'codename': 'bionic'}
 
 
+class FakeDatasource:
+    """Fake Datasource helper object"""
+    def __init__(self):
+        self.region = 'region'
+
+
+class FakeCloud:
+    """Fake Cloud helper object"""
+    def __init__(self):
+        self.datasource = FakeDatasource()
+
+
 class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
     """TestAptSourceConfig
     Main Class to test apt configs
@@ -471,7 +483,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
         fromfn = ("%s/%s_%s" % (pre, archive, post))
         tofn = ("%s/test.ubuntu.com_%s" % (pre, post))
 
-        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch)
+        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, FakeCloud(), arch)
 
         self.assertEqual(mirrors['MIRROR'],
                          "http://test.ubuntu.com/%s/" % component)
@@ -559,7 +571,8 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
                "security": [{'arches': ["default"],
                              "uri": smir}]}
 
-        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, 'amd64')
+        mirrors = cc_apt_configure.find_apt_mirror_info(
+            cfg, FakeCloud(), 'amd64')
 
         self.assertEqual(mirrors['MIRROR'],
                          pmir)
@@ -594,7 +607,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
                "security": [{'arches': ["default"], "uri": "nothis-security"},
                             {'arches': [arch], "uri": smir}]}
 
-        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch)
+        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, FakeCloud(), arch)
 
         self.assertEqual(mirrors['PRIMARY'], pmir)
         self.assertEqual(mirrors['MIRROR'], pmir)
@@ -613,7 +626,8 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
                             {'arches': ["default"],
                              "uri": smir}]}
 
-        mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, 'amd64')
+        mirrors = cc_apt_configure.find_apt_mirror_info(
+            cfg, FakeCloud(), 'amd64')
 
         self.assertEqual(mirrors['MIRROR'],
                          pmir)
@@ -673,7 +687,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
 
         with mock.patch.object(cc_apt_configure.util, 'search_for_mirror',
                                side_effect=[pmir, smir]) as mocksearch:
-            mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None,
+            mirrors = cc_apt_configure.find_apt_mirror_info(cfg, FakeCloud(),
                                                             'amd64')
 
         calls = [call(["pfailme", pmir]),
@@ -712,7 +726,8 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase):
         # should not be called, since primary is specified
         with mock.patch.object(cc_apt_configure.util,
                                'search_for_mirror') as mockse:
-            mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch)
+            mirrors = cc_apt_configure.find_apt_mirror_info(
+                cfg, FakeCloud(), arch)
         mockse.assert_not_called()
 
         self.assertEqual(mirrors['MIRROR'],
-- 
cgit v1.2.3


From 3cec3881062490727c5fff1b16b53f0176f976f0 Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Mon, 13 Jul 2020 12:00:32 -0400
Subject: cloudinit: remove global disable of pylint W0105 and fix errors
 (#480)

This includes a fix to a test that had a string concatenation issue, and
so was only testing a prefix of what was intended.
---
 .pylintrc                                          |  3 +-
 cloudinit/analyze/show.py                          | 41 ++++++++++------------
 cloudinit/config/cc_resizefs.py                    | 14 ++++----
 cloudinit/distros/opensuse.py                      |  2 +-
 cloudinit/net/eni.py                               |  6 ++--
 cloudinit/sources/DataSourceAzure.py               | 14 ++++----
 cloudinit/sources/DataSourceHetzner.py             |  6 ++--
 .../unittests/test_datasource/test_azure_helper.py |  7 ++--
 tests/unittests/test_reporting_hyperv.py           |  6 ++--
 9 files changed, 45 insertions(+), 54 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/.pylintrc b/.pylintrc
index 4d5d066d..f04603b9 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -7,7 +7,6 @@ jobs=4
 [MESSAGES CONTROL]
 
 # Errors and warings with some filtered:
-# W0105(pointless-string-statement)
 # W0107(unnecessary-pass)
 # W0201(attribute-defined-outside-init)
 # W0212(protected-access)
@@ -28,7 +27,7 @@ jobs=4
 # W0703(broad-except)
 # W1401(anomalous-backslash-in-string)
 
-disable=C, F, I, R, W0105, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401
+disable=C, F, I, R, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401
 
 
 [REPORTS]
diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py
index cca1fa7f..0c825b23 100644
--- a/cloudinit/analyze/show.py
+++ b/cloudinit/analyze/show.py
@@ -15,28 +15,25 @@ from cloudinit import subp
 from cloudinit import util
 from cloudinit.distros import uses_systemd
 
-#  An event:
-'''
-{
-        "description": "executing late commands",
-        "event_type": "start",
-        "level": "INFO",
-        "name": "cmd-install/stage-late"
-        "origin": "cloudinit",
-        "timestamp": 1461164249.1590767,
-},
-
-    {
-        "description": "executing late commands",
-        "event_type": "finish",
-        "level": "INFO",
-        "name": "cmd-install/stage-late",
-        "origin": "cloudinit",
-        "result": "SUCCESS",
-        "timestamp": 1461164249.1590767
-    }
-
-'''
+# Example events:
+#     {
+#             "description": "executing late commands",
+#             "event_type": "start",
+#             "level": "INFO",
+#             "name": "cmd-install/stage-late"
+#             "origin": "cloudinit",
+#             "timestamp": 1461164249.1590767,
+#     }
+#     {
+#         "description": "executing late commands",
+#         "event_type": "finish",
+#         "level": "INFO",
+#         "name": "cmd-install/stage-late",
+#         "origin": "cloudinit",
+#         "result": "SUCCESS",
+#         "timestamp": 1461164249.1590767
+#     }
+
 format_key = {
     '%d': 'delta',
     '%D': 'description',
diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py
index 8de4db30..978d2ee0 100644
--- a/cloudinit/config/cc_resizefs.py
+++ b/cloudinit/config/cc_resizefs.py
@@ -118,14 +118,12 @@ def _can_skip_resize_ufs(mount_point, devpth):
                 if o == "-f":
                     frag_sz = int(a)
     # check the current partition size
-    """
-    # gpart show /dev/da0
-=>      40  62914480  da0  GPT  (30G)
-        40      1024    1  freebsd-boot  (512K)
-      1064  58719232    2  freebsd-ufs  (28G)
-  58720296   3145728    3  freebsd-swap  (1.5G)
-  61866024   1048496       - free -  (512M)
-    """
+    # Example output from `gpart show /dev/da0`:
+    # =>      40  62914480  da0  GPT  (30G)
+    #         40      1024    1  freebsd-boot  (512K)
+    #       1064  58719232    2  freebsd-ufs  (28G)
+    #   58720296   3145728    3  freebsd-swap  (1.5G)
+    #   61866024   1048496       - free -  (512M)
     expect_sz = None
     m = re.search('^(/dev/.+)p([0-9])$', devpth)
     gpart_res = _get_gpart_output(m.group(1))
diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py
index ffb7d0e8..b8e557b8 100644
--- a/cloudinit/distros/opensuse.py
+++ b/cloudinit/distros/opensuse.py
@@ -185,7 +185,7 @@ class Distro(distros.Distro):
     def preferred_ntp_clients(self):
         """The preferred ntp client is dependent on the version."""
 
-        """Allow distro to determine the preferred ntp client list"""
+        # Allow distro to determine the preferred ntp client list
         if not self._preferred_ntp_clients:
             distro_info = util.system_info()['dist']
             name = distro_info[0]
diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py
index b4c69457..13c041f3 100644
--- a/cloudinit/net/eni.py
+++ b/cloudinit/net/eni.py
@@ -483,10 +483,8 @@ class Renderer(renderer.Renderer):
         if searchdomains:
             lo['subnets'][0]["dns_search"] = (" ".join(searchdomains))
 
-        ''' Apply a sort order to ensure that we write out
-            the physical interfaces first; this is critical for
-            bonding
-        '''
+        # Apply a sort order to ensure that we write out the physical
+        # interfaces first; this is critical for bonding
         order = {
             'loopback': 0,
             'physical': 1,
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 41431a7a..5e25b956 100755
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -166,12 +166,11 @@ def get_resource_disk_on_freebsd(port_id):
         port_id = port_id - 2
     g1 = "000" + str(port_id)
     g0g1 = "{0}-{1}".format(g0, g1)
-    """
-    search 'X' from
-       'dev.storvsc.X.%pnpinfo:
-           classid=32412632-86cb-44a2-9b5c-50d1417354f5
-           deviceid=00000000-0001-8899-0000-000000000000'
-    """
+
+    # search 'X' from
+    #  'dev.storvsc.X.%pnpinfo:
+    #      classid=32412632-86cb-44a2-9b5c-50d1417354f5
+    #      deviceid=00000000-0001-8899-0000-000000000000'
     sysctl_out = get_dev_storvsc_sysctl()
 
     storvscid = find_storvscid_from_sysctl_pnpinfo(sysctl_out, g0g1)
@@ -1485,13 +1484,12 @@ def maybe_remove_ubuntu_network_config_scripts(paths=None):
 
 
 def _is_platform_viable(seed_dir):
+    """Check platform environment to report if this datasource may run."""
     with events.ReportEventStack(
         name="check-platform-viability",
         description="found azure asset tag",
         parent=azure_ds_reporter
     ) as evt:
-
-        """Check platform environment to report if this datasource may run."""
         asset_tag = util.read_dmi_data('chassis-asset-tag')
         if asset_tag == AZURE_CHASSIS_ASSET_TAG:
             return True
diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py
index 70e4274c..a86035e0 100644
--- a/cloudinit/sources/DataSourceHetzner.py
+++ b/cloudinit/sources/DataSourceHetzner.py
@@ -69,9 +69,9 @@ class DataSourceHetzner(sources.DataSource):
         self.userdata_raw = hc_helper.maybe_b64decode(ud)
         self.metadata_full = md
 
-        """hostname is name provided by user at launch.  The API enforces
-        it is a valid hostname, but it is not guaranteed to be resolvable
-        in dns or fully qualified."""
+        # hostname is name provided by user at launch.  The API enforces it is
+        # a valid hostname, but it is not guaranteed to be resolvable in dns or
+        # fully qualified.
         self.metadata['instance-id'] = md['instance-id']
         self.metadata['local-hostname'] = md['hostname']
         self.metadata['network-config'] = md.get('network-config', None)
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index 71ef57f0..f314cd4a 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -404,11 +404,10 @@ class TestWALinuxAgentShim(CiTestCase):
             self.GoalState.call_args_list)
 
     def test_certificates_used_to_determine_public_keys(self):
+        # if register_with_azure_and_fetch_data() isn't passed some info about
+        # the user's public keys, there's no point in even trying to parse the
+        # certificates
         shim = wa_shim()
-        """if register_with_azure_and_fetch_data() isn't passed some info about
-           the user's public keys, there's no point in even trying to parse
-           the certificates
-        """
         mypk = [{'fingerprint': 'fp1', 'path': 'path1'},
                 {'fingerprint': 'fp3', 'path': 'path3', 'value': ''}]
         certs = {'fp1': 'expected-key',
diff --git a/tests/unittests/test_reporting_hyperv.py b/tests/unittests/test_reporting_hyperv.py
index b60a66ab..bacf5da9 100644
--- a/tests/unittests/test_reporting_hyperv.py
+++ b/tests/unittests/test_reporting_hyperv.py
@@ -139,8 +139,10 @@ class TextKvpReporter(CiTestCase):
     @mock.patch('cloudinit.subp.subp')
     def test_get_boot_telemetry(self, m_subp, m_sysd):
         reporter = HyperVKvpReportingHandler(kvp_file_path=self.tmp_file_path)
-        datetime_pattern = r"\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]"
-        r"\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z)"
+        datetime_pattern = (
+            r"\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]"
+            r"\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z)"
+        )
 
         # get_boot_telemetry makes two subp calls to systemctl. We provide
         # a list of values that the subp calls should return
-- 
cgit v1.2.3


From 25289087e44c9c74543248519e37ca1f11b8a711 Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Tue, 14 Jul 2020 13:31:13 -0400
Subject: networking: refactor wait_for_physdevs from cloudinit.net (#466)

* Refactor `cloudinit.net.wait_for_physdevs` to `cloudinit.distros.networking.Networking.wait_for_physdevs`
* Split the Linux-specific `udevadm_settle` call out to a separate abstract `Networking.settle` method; implement it on `LinuxNetworking` and add a `NotImplementedError` implementation to `BSDNetworking`
* Modify `wait_for_physdevs`s one callsite to use the new location

LP: #1884626
---
 cloudinit/distros/networking.py            |  79 ++++++++++++++-
 cloudinit/distros/tests/test_networking.py | 152 ++++++++++++++++++++++++++++-
 cloudinit/net/__init__.py                  |  38 --------
 cloudinit/net/tests/test_init.py           |  74 --------------
 cloudinit/stages.py                        |   2 +-
 5 files changed, 229 insertions(+), 116 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/networking.py b/cloudinit/distros/networking.py
index e421a2ce..75e69df5 100644
--- a/cloudinit/distros/networking.py
+++ b/cloudinit/distros/networking.py
@@ -1,7 +1,11 @@
 import abc
+import logging
 import os
 
-from cloudinit import net
+from cloudinit import net, util
+
+
+LOG = logging.getLogger(__name__)
 
 
 # Type aliases (https://docs.python.org/3/library/typing.html#type-aliases),
@@ -102,10 +106,72 @@ class Networking(metaclass=abc.ABCMeta):
     def master_is_bridge_or_bond(self, devname: DeviceName) -> bool:
         return net.master_is_bridge_or_bond(devname)
 
+    @abc.abstractmethod
+    def settle(self, *, exists=None) -> None:
+        """Wait for device population in the system to complete.
+
+        :param exists:
+            An optional optimisation.  If given, only perform as much of the
+            settle process as is required for the given DeviceName to be
+            present in the system.  (This may include skipping the settle
+            process entirely, if the device already exists.)
+        :type exists: Optional[DeviceName]
+        """
+        pass
+
     def wait_for_physdevs(
         self, netcfg: NetworkConfig, *, strict: bool = True
     ) -> None:
-        return net.wait_for_physdevs(netcfg, strict=strict)
+        """Wait for all the physical devices in `netcfg` to exist on the system
+
+        Specifically, this will call `self.settle` 5 times, and check after
+        each one if the physical devices are now present in the system.
+
+        :param netcfg:
+            The NetworkConfig from which to extract physical devices to wait
+            for.
+        :param strict:
+            Raise a `RuntimeError` if any physical devices are not present
+            after waiting.
+        """
+        physdevs = self.extract_physdevs(netcfg)
+
+        # set of expected iface names and mac addrs
+        expected_ifaces = dict([(iface[0], iface[1]) for iface in physdevs])
+        expected_macs = set(expected_ifaces.keys())
+
+        # set of current macs
+        present_macs = self.get_interfaces_by_mac().keys()
+
+        # compare the set of expected mac address values to
+        # the current macs present; we only check MAC as cloud-init
+        # has not yet renamed interfaces and the netcfg may include
+        # such renames.
+        for _ in range(0, 5):
+            if expected_macs.issubset(present_macs):
+                LOG.debug("net: all expected physical devices present")
+                return
+
+            missing = expected_macs.difference(present_macs)
+            LOG.debug("net: waiting for expected net devices: %s", missing)
+            for mac in missing:
+                # trigger a settle, unless this interface exists
+                devname = expected_ifaces[mac]
+                msg = "Waiting for settle or {} exists".format(devname)
+                util.log_time(
+                    LOG.debug,
+                    msg,
+                    func=self.settle,
+                    kwargs={"exists": devname},
+                )
+
+            # update present_macs after settles
+            present_macs = self.get_interfaces_by_mac().keys()
+
+        msg = "Not all expected physical devices present: %s" % missing
+        LOG.warning(msg)
+        if strict:
+            raise RuntimeError(msg)
 
 
 class BSDNetworking(Networking):
@@ -114,6 +180,10 @@ class BSDNetworking(Networking):
     def is_physical(self, devname: DeviceName) -> bool:
         raise NotImplementedError()
 
+    def settle(self, *, exists=None) -> None:
+        """BSD has no equivalent to `udevadm settle`; noop."""
+        pass
+
 
 class LinuxNetworking(Networking):
     """Implementation of networking functionality common to Linux distros."""
@@ -138,3 +208,8 @@ class LinuxNetworking(Networking):
 
     def is_physical(self, devname: DeviceName) -> bool:
         return os.path.exists(net.sys_dev_path(devname, "device"))
+
+    def settle(self, *, exists=None) -> None:
+        if exists is not None:
+            exists = net.sys_dev_path(exists)
+        util.udevadm_settle(exists=exists)
diff --git a/cloudinit/distros/tests/test_networking.py b/cloudinit/distros/tests/test_networking.py
index 2acb12f4..b9a63842 100644
--- a/cloudinit/distros/tests/test_networking.py
+++ b/cloudinit/distros/tests/test_networking.py
@@ -2,7 +2,39 @@ from unittest import mock
 
 import pytest
 
-from cloudinit.distros.networking import BSDNetworking, LinuxNetworking
+from cloudinit import net
+from cloudinit.distros.networking import (
+    BSDNetworking,
+    LinuxNetworking,
+    Networking,
+)
+
+# See https://docs.pytest.org/en/stable/example
+# /parametrize.html#parametrizing-conditional-raising
+from contextlib import ExitStack as does_not_raise
+
+
+@pytest.yield_fixture
+def generic_networking_cls():
+    """Returns a direct Networking subclass which errors on /sys usage.
+
+    This enables the direct testing of functionality only present on the
+    ``Networking`` super-class, and provides a check on accidentally using /sys
+    in that context.
+    """
+
+    class TestNetworking(Networking):
+        def is_physical(self, *args, **kwargs):
+            raise NotImplementedError
+
+        def settle(self, *args, **kwargs):
+            raise NotImplementedError
+
+    error = AssertionError("Unexpectedly used /sys in generic networking code")
+    with mock.patch(
+        "cloudinit.net.get_sys_class_path", side_effect=error,
+    ):
+        yield TestNetworking
 
 
 @pytest.yield_fixture
@@ -40,3 +72,121 @@ class TestLinuxNetworkingIsPhysical:
         device_dir.join("device").write("")
 
         assert LinuxNetworking().is_physical(devname)
+
+
+class TestBSDNetworkingSettle:
+    def test_settle_doesnt_error(self):
+        # This also implicitly tests that it doesn't use subp.subp
+        BSDNetworking().settle()
+
+
+@pytest.mark.usefixtures("sys_class_net")
+@mock.patch("cloudinit.distros.networking.util.udevadm_settle", autospec=True)
+class TestLinuxNetworkingSettle:
+    def test_no_arguments(self, m_udevadm_settle):
+        LinuxNetworking().settle()
+
+        assert [mock.call(exists=None)] == m_udevadm_settle.call_args_list
+
+    def test_exists_argument(self, m_udevadm_settle):
+        LinuxNetworking().settle(exists="ens3")
+
+        expected_path = net.sys_dev_path("ens3")
+        assert [
+            mock.call(exists=expected_path)
+        ] == m_udevadm_settle.call_args_list
+
+
+class TestNetworkingWaitForPhysDevs:
+    @pytest.fixture
+    def wait_for_physdevs_netcfg(self):
+        """This config is shared across all the tests in this class."""
+
+        def ethernet(mac, name, driver=None, device_id=None):
+            v2_cfg = {"set-name": name, "match": {"macaddress": mac}}
+            if driver:
+                v2_cfg["match"].update({"driver": driver})
+            if device_id:
+                v2_cfg["match"].update({"device_id": device_id})
+
+            return v2_cfg
+
+        physdevs = [
+            ["aa:bb:cc:dd:ee:ff", "eth0", "virtio", "0x1000"],
+            ["00:11:22:33:44:55", "ens3", "e1000", "0x1643"],
+        ]
+        netcfg = {
+            "version": 2,
+            "ethernets": {args[1]: ethernet(*args) for args in physdevs},
+        }
+        return netcfg
+
+    def test_skips_settle_if_all_present(
+        self, generic_networking_cls, wait_for_physdevs_netcfg,
+    ):
+        networking = generic_networking_cls()
+        with mock.patch.object(
+            networking, "get_interfaces_by_mac"
+        ) as m_get_interfaces_by_mac:
+            m_get_interfaces_by_mac.side_effect = iter(
+                [{"aa:bb:cc:dd:ee:ff": "eth0", "00:11:22:33:44:55": "ens3"}]
+            )
+            with mock.patch.object(
+                networking, "settle", autospec=True
+            ) as m_settle:
+                networking.wait_for_physdevs(wait_for_physdevs_netcfg)
+            assert 0 == m_settle.call_count
+
+    def test_calls_udev_settle_on_missing(
+        self, generic_networking_cls, wait_for_physdevs_netcfg,
+    ):
+        networking = generic_networking_cls()
+        with mock.patch.object(
+            networking, "get_interfaces_by_mac"
+        ) as m_get_interfaces_by_mac:
+            m_get_interfaces_by_mac.side_effect = iter(
+                [
+                    {
+                        "aa:bb:cc:dd:ee:ff": "eth0"
+                    },  # first call ens3 is missing
+                    {
+                        "aa:bb:cc:dd:ee:ff": "eth0",
+                        "00:11:22:33:44:55": "ens3",
+                    },  # second call has both
+                ]
+            )
+            with mock.patch.object(
+                networking, "settle", autospec=True
+            ) as m_settle:
+                networking.wait_for_physdevs(wait_for_physdevs_netcfg)
+            m_settle.assert_called_with(exists="ens3")
+
+    @pytest.mark.parametrize(
+        "strict,expectation",
+        [(True, pytest.raises(RuntimeError)), (False, does_not_raise())],
+    )
+    def test_retrying_and_strict_behaviour(
+        self,
+        strict,
+        expectation,
+        generic_networking_cls,
+        wait_for_physdevs_netcfg,
+    ):
+        networking = generic_networking_cls()
+        with mock.patch.object(
+            networking, "get_interfaces_by_mac"
+        ) as m_get_interfaces_by_mac:
+            m_get_interfaces_by_mac.return_value = {}
+
+            with mock.patch.object(
+                networking, "settle", autospec=True
+            ) as m_settle:
+                with expectation:
+                    networking.wait_for_physdevs(
+                        wait_for_physdevs_netcfg, strict=strict
+                    )
+
+        assert (
+            5 * len(wait_for_physdevs_netcfg["ethernets"])
+            == m_settle.call_count
+        )
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 9d8c7ba9..322af77b 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -10,7 +10,6 @@ import ipaddress
 import logging
 import os
 import re
-from functools import partial
 
 from cloudinit import subp
 from cloudinit import util
@@ -494,43 +493,6 @@ def extract_physdevs(netcfg):
     raise RuntimeError('Unknown network config version: %s' % version)
 
 
-def wait_for_physdevs(netcfg, strict=True):
-    physdevs = extract_physdevs(netcfg)
-
-    # set of expected iface names and mac addrs
-    expected_ifaces = dict([(iface[0], iface[1]) for iface in physdevs])
-    expected_macs = set(expected_ifaces.keys())
-
-    # set of current macs
-    present_macs = get_interfaces_by_mac().keys()
-
-    # compare the set of expected mac address values to
-    # the current macs present; we only check MAC as cloud-init
-    # has not yet renamed interfaces and the netcfg may include
-    # such renames.
-    for _ in range(0, 5):
-        if expected_macs.issubset(present_macs):
-            LOG.debug('net: all expected physical devices present')
-            return
-
-        missing = expected_macs.difference(present_macs)
-        LOG.debug('net: waiting for expected net devices: %s', missing)
-        for mac in missing:
-            # trigger a settle, unless this interface exists
-            syspath = sys_dev_path(expected_ifaces[mac])
-            settle = partial(util.udevadm_settle, exists=syspath)
-            msg = 'Waiting for udev events to settle or %s exists' % syspath
-            util.log_time(LOG.debug, msg, func=settle)
-
-        # update present_macs after settles
-        present_macs = get_interfaces_by_mac().keys()
-
-    msg = 'Not all expected physical devices present: %s' % missing
-    LOG.warning(msg)
-    if strict:
-        raise RuntimeError(msg)
-
-
 def apply_network_config_names(netcfg, strict_present=True, strict_busy=True):
     """read the network config and rename devices accordingly.
     if strict_present is false, then do not raise exception if no devices
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index eb458c39..311ab6f8 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -963,80 +963,6 @@ class TestExtractPhysdevs(CiTestCase):
             net.extract_physdevs({'version': 3, 'awesome_config': []})
 
 
-class TestWaitForPhysdevs(CiTestCase):
-
-    def setUp(self):
-        super(TestWaitForPhysdevs, self).setUp()
-        self.add_patch('cloudinit.net.get_interfaces_by_mac',
-                       'm_get_iface_mac')
-        self.add_patch('cloudinit.util.udevadm_settle', 'm_udev_settle')
-
-    def test_wait_for_physdevs_skips_settle_if_all_present(self):
-        physdevs = [
-            ['aa:bb:cc:dd:ee:ff', 'eth0', 'virtio', '0x1000'],
-            ['00:11:22:33:44:55', 'ens3', 'e1000', '0x1643'],
-        ]
-        netcfg = {
-            'version': 2,
-            'ethernets': {args[1]: _mk_v2_phys(*args)
-                          for args in physdevs},
-        }
-        self.m_get_iface_mac.side_effect = iter([
-            {'aa:bb:cc:dd:ee:ff': 'eth0',
-             '00:11:22:33:44:55': 'ens3'},
-        ])
-        net.wait_for_physdevs(netcfg)
-        self.assertEqual(0, self.m_udev_settle.call_count)
-
-    def test_wait_for_physdevs_calls_udev_settle_on_missing(self):
-        physdevs = [
-            ['aa:bb:cc:dd:ee:ff', 'eth0', 'virtio', '0x1000'],
-            ['00:11:22:33:44:55', 'ens3', 'e1000', '0x1643'],
-        ]
-        netcfg = {
-            'version': 2,
-            'ethernets': {args[1]: _mk_v2_phys(*args)
-                          for args in physdevs},
-        }
-        self.m_get_iface_mac.side_effect = iter([
-            {'aa:bb:cc:dd:ee:ff': 'eth0'},   # first call ens3 is missing
-            {'aa:bb:cc:dd:ee:ff': 'eth0',
-             '00:11:22:33:44:55': 'ens3'},   # second call has both
-        ])
-        net.wait_for_physdevs(netcfg)
-        self.m_udev_settle.assert_called_with(exists=net.sys_dev_path('ens3'))
-
-    def test_wait_for_physdevs_raise_runtime_error_if_missing_and_strict(self):
-        physdevs = [
-            ['aa:bb:cc:dd:ee:ff', 'eth0', 'virtio', '0x1000'],
-            ['00:11:22:33:44:55', 'ens3', 'e1000', '0x1643'],
-        ]
-        netcfg = {
-            'version': 2,
-            'ethernets': {args[1]: _mk_v2_phys(*args)
-                          for args in physdevs},
-        }
-        self.m_get_iface_mac.return_value = {}
-        with self.assertRaises(RuntimeError):
-            net.wait_for_physdevs(netcfg)
-
-        self.assertEqual(5 * len(physdevs), self.m_udev_settle.call_count)
-
-    def test_wait_for_physdevs_no_raise_if_not_strict(self):
-        physdevs = [
-            ['aa:bb:cc:dd:ee:ff', 'eth0', 'virtio', '0x1000'],
-            ['00:11:22:33:44:55', 'ens3', 'e1000', '0x1643'],
-        ]
-        netcfg = {
-            'version': 2,
-            'ethernets': {args[1]: _mk_v2_phys(*args)
-                          for args in physdevs},
-        }
-        self.m_get_iface_mac.return_value = {}
-        net.wait_for_physdevs(netcfg, strict=False)
-        self.assertEqual(5 * len(physdevs), self.m_udev_settle.call_count)
-
-
 class TestNetFailOver(CiTestCase):
 
     def setUp(self):
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index db8ba64c..69e6b7e1 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -696,7 +696,7 @@ class Init(object):
                     netcfg, src = self._find_networking_config()
 
         # ensure all physical devices in config are present
-        net.wait_for_physdevs(netcfg)
+        self.distro.networking.wait_for_physdevs(netcfg)
 
         # apply renames from config
         self._apply_netcfg_names(netcfg)
-- 
cgit v1.2.3


From 4fe576516d65feda17ba78e9265a8e494a195e7b Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Wed, 15 Jul 2020 10:26:12 -0400
Subject: cloudinit: remove global disable of pylint W0107 and fix errors
 (#489)

* cloudinit: remove global disable of pylint W0107 and fix errors

This includes removing a test class which contained no tests but wasn't
detected as empty because of an errant pass statement.

* .pylintrc: update disable comment to match arguments
---
 .pylintrc                                                | 4 +---
 cloudinit/config/cc_set_hostname.py                      | 1 -
 cloudinit/distros/networking.py                          | 3 ---
 cloudinit/distros/ubuntu.py                              | 2 --
 cloudinit/net/cmdline.py                                 | 2 --
 cloudinit/net/dhcp.py                                    | 2 --
 cloudinit/reporting/handlers.py                          | 1 -
 cloudinit/sources/DataSourceAzure.py                     | 1 -
 cloudinit/sources/__init__.py                            | 2 --
 cloudinit/sources/helpers/netlink.py                     | 1 -
 cloudinit/sources/helpers/vmware/imc/config_file.py      | 1 -
 cloudinit/sources/helpers/vmware/imc/config_namespace.py | 1 -
 cloudinit/sources/helpers/vmware/imc/config_source.py    | 1 -
 cloudinit/stages.py                                      | 1 -
 tests/cloud_tests/platforms/azurecloud/instance.py       | 2 --
 tests/cloud_tests/platforms/images.py                    | 1 -
 tests/cloud_tests/platforms/snapshots.py                 | 1 -
 tests/cloud_tests/testcases/base.py                      | 1 -
 tests/unittests/test_datasource/test_ibmcloud.py         | 7 -------
 tests/unittests/test_reporting.py                        | 1 -
 20 files changed, 1 insertion(+), 35 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/.pylintrc b/.pylintrc
index f04603b9..94a81d0e 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -7,7 +7,6 @@ jobs=4
 [MESSAGES CONTROL]
 
 # Errors and warings with some filtered:
-# W0107(unnecessary-pass)
 # W0201(attribute-defined-outside-init)
 # W0212(protected-access)
 # W0221(arguments-differ)
@@ -19,7 +18,6 @@ jobs=4
 # W0602(global-variable-not-assigned)
 # W0603(global-statement)
 # W0611(unused-import)
-# W0612(unused-variable)
 # W0613(unused-argument)
 # W0621(redefined-outer-name)
 # W0622(redefined-builtin)
@@ -27,7 +25,7 @@ jobs=4
 # W0703(broad-except)
 # W1401(anomalous-backslash-in-string)
 
-disable=C, F, I, R, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401
+disable=C, F, I, R, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401
 
 
 [REPORTS]
diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py
index 10d6d197..c13020d8 100644
--- a/cloudinit/config/cc_set_hostname.py
+++ b/cloudinit/config/cc_set_hostname.py
@@ -55,7 +55,6 @@ class SetHostnameError(Exception):
     This may happen if we attempt to set the hostname early in cloud-init's
     init-local timeframe as certain services may not be running yet.
     """
-    pass
 
 
 def handle(name, cfg, cloud, log, _args):
diff --git a/cloudinit/distros/networking.py b/cloudinit/distros/networking.py
index 75e69df5..10ed249d 100644
--- a/cloudinit/distros/networking.py
+++ b/cloudinit/distros/networking.py
@@ -92,7 +92,6 @@ class Networking(metaclass=abc.ABCMeta):
         Examples of non-physical network devices: bonds, bridges, tunnels,
         loopback devices.
         """
-        pass
 
     def is_renamed(self, devname: DeviceName) -> bool:
         return net.is_renamed(devname)
@@ -117,7 +116,6 @@ class Networking(metaclass=abc.ABCMeta):
             process entirely, if the device already exists.)
         :type exists: Optional[DeviceName]
         """
-        pass
 
     def wait_for_physdevs(
         self, netcfg: NetworkConfig, *, strict: bool = True
@@ -182,7 +180,6 @@ class BSDNetworking(Networking):
 
     def settle(self, *, exists=None) -> None:
         """BSD has no equivalent to `udevadm settle`; noop."""
-        pass
 
 
 class LinuxNetworking(Networking):
diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py
index 23be3bdd..b4c4b0c3 100644
--- a/cloudinit/distros/ubuntu.py
+++ b/cloudinit/distros/ubuntu.py
@@ -49,7 +49,5 @@ class Distro(debian.Distro):
                     copy.deepcopy(PREFERRED_NTP_CLIENTS))
         return self._preferred_ntp_clients
 
-    pass
-
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py
index 0e83685d..7ca7262b 100755
--- a/cloudinit/net/cmdline.py
+++ b/cloudinit/net/cmdline.py
@@ -29,12 +29,10 @@ class InitramfsNetworkConfigSource(metaclass=abc.ABCMeta):
     @abc.abstractmethod
     def is_applicable(self) -> bool:
         """Is this initramfs config source applicable to the current system?"""
-        pass
 
     @abc.abstractmethod
     def render_config(self) -> dict:
         """Render a v1 network config from the initramfs configuration"""
-        pass
 
 
 class KlibcNetworkConfigSource(InitramfsNetworkConfigSource):
diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py
index 9e004688..d1d1255e 100644
--- a/cloudinit/net/dhcp.py
+++ b/cloudinit/net/dhcp.py
@@ -31,12 +31,10 @@ class InvalidDHCPLeaseFileError(Exception):
     Current uses are DataSourceAzure and DataSourceEc2 during ephemeral
     boot to scrape metadata.
     """
-    pass
 
 
 class NoDHCPLeaseError(Exception):
     """Raised when unable to get a DHCP lease."""
-    pass
 
 
 class EphemeralDHCPv4(object):
diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py
index 6b9127b6..1986ebdd 100755
--- a/cloudinit/reporting/handlers.py
+++ b/cloudinit/reporting/handlers.py
@@ -35,7 +35,6 @@ class ReportingHandler(metaclass=abc.ABCMeta):
 
     def flush(self):
         """Ensure ReportingHandler has published all events"""
-        pass
 
 
 class LogHandler(ReportingHandler):
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 5e25b956..8b34d047 100755
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -688,7 +688,6 @@ class DataSourceAzure(sources.DataSource):
             except UrlError:
                 # Teardown our EphemeralDHCPv4 context on failure as we retry
                 self._ephemeral_dhcp_ctx.clean_network()
-                pass
             finally:
                 if nl_sock:
                     nl_sock.close()
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 923e3cea..c4d60fff 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -78,7 +78,6 @@ class DataSourceNotFoundException(Exception):
 
 class InvalidMetaDataException(Exception):
     """Raised when metadata is broken, unavailable or disabled."""
-    pass
 
 
 def process_instance_metadata(metadata, key_path='', sensitive_keys=()):
@@ -511,7 +510,6 @@ class DataSource(metaclass=abc.ABCMeta):
             (e.g. 'ssh-rsa') and key_value is the key itself
             (e.g. 'AAAAB3NzaC1y...').
         """
-        pass
 
     def _remap_device(self, short_name):
         # LP: #611137
diff --git a/cloudinit/sources/helpers/netlink.py b/cloudinit/sources/helpers/netlink.py
index d377ae3d..a74a3588 100644
--- a/cloudinit/sources/helpers/netlink.py
+++ b/cloudinit/sources/helpers/netlink.py
@@ -55,7 +55,6 @@ NetlinkHeader = namedtuple('NetlinkHeader', ['length', 'type', 'flags', 'seq',
 
 class NetlinkCreateSocketError(RuntimeError):
     '''Raised if netlink socket fails during create or bind.'''
-    pass
 
 
 def create_bound_netlink_socket():
diff --git a/cloudinit/sources/helpers/vmware/imc/config_file.py b/cloudinit/sources/helpers/vmware/imc/config_file.py
index 602af078..fc034c95 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_file.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_file.py
@@ -22,7 +22,6 @@ class ConfigFile(ConfigSource, dict):
 
     def __init__(self, filename):
         self._loadConfigFile(filename)
-        pass
 
     def _insertKey(self, key, val):
         """
diff --git a/cloudinit/sources/helpers/vmware/imc/config_namespace.py b/cloudinit/sources/helpers/vmware/imc/config_namespace.py
index 2f29edd4..5899d8f7 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_namespace.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_namespace.py
@@ -10,6 +10,5 @@ from .config_source import ConfigSource
 
 class ConfigNamespace(ConfigSource):
     """Specifies the Config Namespace."""
-    pass
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py
index 2f8ea546..7ec06a9c 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_source.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_source.py
@@ -8,6 +8,5 @@
 
 class ConfigSource(object):
     """Specifies a source for the Config Content."""
-    pass
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 69e6b7e1..765f4aab 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -947,7 +947,6 @@ def _pkl_load(fname):
     except Exception as e:
         if os.path.isfile(fname):
             LOG.warning("failed loading pickle in %s: %s", fname, e)
-        pass
 
     # This is allowed so just return nothing successfully loaded...
     if not pickle_contents:
diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py
index f1e28a96..a136cf0d 100644
--- a/tests/cloud_tests/platforms/azurecloud/instance.py
+++ b/tests/cloud_tests/platforms/azurecloud/instance.py
@@ -80,7 +80,6 @@ class AzureCloudInstance(Instance):
         except CloudError:
             LOG.debug(('image not found, launching instance with base image, '
                        'image_id=%s'), self.image_id)
-            pass
 
         vm_params = {
             'name': self.vm_name,
@@ -169,7 +168,6 @@ class AzureCloudInstance(Instance):
                     sleep(15)
                 else:
                     LOG.warning('Could not find console log: %s', e)
-                    pass
 
         LOG.debug('stopping instance %s', self.image_id)
         vm_deallocate = \
diff --git a/tests/cloud_tests/platforms/images.py b/tests/cloud_tests/platforms/images.py
index 557a5cf6..f047de2e 100644
--- a/tests/cloud_tests/platforms/images.py
+++ b/tests/cloud_tests/platforms/images.py
@@ -52,6 +52,5 @@ class Image(TargetBase):
 
     def destroy(self):
         """Clean up data associated with image."""
-        pass
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/platforms/snapshots.py b/tests/cloud_tests/platforms/snapshots.py
index 94328982..0f5f8bb6 100644
--- a/tests/cloud_tests/platforms/snapshots.py
+++ b/tests/cloud_tests/platforms/snapshots.py
@@ -40,6 +40,5 @@ class Snapshot(object):
 
     def destroy(self):
         """Clean up snapshot data."""
-        pass
 
 # vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index 2e7c6686..4448e0b5 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -34,7 +34,6 @@ class CloudTestCase(unittest.TestCase):
     @classmethod
     def maybeSkipTest(cls):
         """Present to allow subclasses to override and raise a skipTest."""
-        pass
 
     def assertPackageInstalled(self, name, version=None):
         """Check dpkg-query --show output for matching package name.
diff --git a/tests/unittests/test_datasource/test_ibmcloud.py b/tests/unittests/test_datasource/test_ibmcloud.py
index 0b54f585..9013ae9f 100644
--- a/tests/unittests/test_datasource/test_ibmcloud.py
+++ b/tests/unittests/test_datasource/test_ibmcloud.py
@@ -15,13 +15,6 @@ mock = test_helpers.mock
 D_PATH = "cloudinit.sources.DataSourceIBMCloud."
 
 
-class TestIBMCloud(test_helpers.CiTestCase):
-    """Test the datasource."""
-    def setUp(self):
-        super(TestIBMCloud, self).setUp()
-        pass
-
-
 @mock.patch(D_PATH + "_is_xen", return_value=True)
 @mock.patch(D_PATH + "_is_ibm_provisioning")
 @mock.patch(D_PATH + "util.blkid")
diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py
index 6814030e..9f11fd5c 100644
--- a/tests/unittests/test_reporting.py
+++ b/tests/unittests/test_reporting.py
@@ -349,7 +349,6 @@ class TestReportingEventStack(TestCase):
         with parent:
             with child:
                 pass
-            pass
         self.assertEqual(report_start.call_count, 0)
         self.assertEqual(report_finish.call_count, 0)
 
-- 
cgit v1.2.3


From 6ac71fdf22db43269cf9b068517290b937ec31b5 Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Thu, 13 Aug 2020 11:27:47 -0400
Subject: cloudinit.distros: update docstrings of add_user and create_user
 (#527)

This aligns their docstrings more closely with their actual behaviour.
---
 cloudinit/distros/__init__.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 2fc91bbc..c7163e1c 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -392,6 +392,9 @@ class Distro(metaclass=abc.ABCMeta):
     def add_user(self, name, **kwargs):
         """
         Add a user to the system using standard GNU tools
+
+        This should be overriden on distros where useradd is not desirable or
+        not available.
         """
         # XXX need to make add_user idempotent somehow as we
         # still want to add groups or modify SSH keys on pre-existing
@@ -520,9 +523,22 @@ class Distro(metaclass=abc.ABCMeta):
 
     def create_user(self, name, **kwargs):
         """
-        Creates users for the system using the GNU passwd tools. This
-        will work on an GNU system. This should be overriden on
-        distros where useradd is not desirable or not available.
+        Creates or partially updates the ``name`` user in the system.
+
+        This defers the actual user creation to ``self.add_user`` or
+        ``self.add_snap_user``, and most of the keys in ``kwargs`` will be
+        processed there if and only if the user does not already exist.
+
+        Once the existence of the ``name`` user has been ensured, this method
+        then processes these keys (for both just-created and pre-existing
+        users):
+
+        * ``plain_text_passwd``
+        * ``hashed_passwd``
+        * ``lock_passwd``
+        * ``sudo``
+        * ``ssh_authorized_keys``
+        * ``ssh_redirect_user``
         """
 
         # Add a snap user, if requested
-- 
cgit v1.2.3


From 79a8ce7e714ae1686c10bff77612eab0f6eccc95 Mon Sep 17 00:00:00 2001
From: dermotbradley <dermot_bradley@yahoo.com>
Date: Thu, 20 Aug 2020 00:18:25 +0100
Subject: Add Alpine Linux support. (#535)

Add new module cc_apk_configure for creating Alpine /etc/apk/repositories file.
Modify cc_ca_certs, cc_ntp, cc_power_state_change, and cc_resolv_conf for Alpine.
Add Alpine template files for Chrony and Busybox NTP support.
Add Alpine template file for /etc/hosts.
---
 README.md                                          |   2 +-
 cloudinit/config/cc_apk_configure.py               | 263 ++++++++++++++++++
 cloudinit/config/cc_ca_certs.py                    |  22 +-
 cloudinit/config/cc_ntp.py                         |  61 ++++-
 cloudinit/config/cc_power_state_change.py          |  57 +++-
 cloudinit/config/cc_resolv_conf.py                 |   4 +-
 cloudinit/distros/__init__.py                      |   7 +-
 cloudinit/distros/alpine.py                        | 165 ++++++++++++
 cloudinit/util.py                                  |   3 +-
 config/cloud.cfg.tmpl                              |  21 +-
 doc/rtd/topics/availability.rst                    |  13 +-
 doc/rtd/topics/instancedata.rst                    |   1 +
 doc/rtd/topics/modules.rst                         |   1 +
 doc/rtd/topics/network-config.rst                  |   2 +-
 templates/chrony.conf.alpine.tmpl                  |  38 +++
 templates/hosts.alpine.tmpl                        |  28 ++
 templates/ntp.conf.alpine.tmpl                     |  10 +
 tests/unittests/test_cli.py                        |   2 +-
 .../test_handler/test_handler_apk_configure.py     | 299 +++++++++++++++++++++
 .../test_handler/test_handler_ca_certs.py          |  11 +-
 tests/unittests/test_handler/test_handler_ntp.py   | 129 ++++++---
 .../test_handler/test_handler_power_state.py       |  29 +-
 tests/unittests/test_handler/test_schema.py        |   1 +
 tools/render-cloudcfg                              |   5 +-
 24 files changed, 1068 insertions(+), 106 deletions(-)
 create mode 100644 cloudinit/config/cc_apk_configure.py
 create mode 100644 cloudinit/distros/alpine.py
 create mode 100644 templates/chrony.conf.alpine.tmpl
 create mode 100644 templates/hosts.alpine.tmpl
 create mode 100644 templates/ntp.conf.alpine.tmpl
 create mode 100644 tests/unittests/test_handler/test_handler_apk_configure.py

(limited to 'cloudinit/distros')

diff --git a/README.md b/README.md
index a3455135..435405da 100644
--- a/README.md
+++ b/README.md
@@ -39,7 +39,7 @@ get in contact with that distribution and send them our way!
 
 | Supported OSes | Supported Public Clouds | Supported Private Clouds |
 | --- | --- | --- |
-| Ubuntu<br />SLES/openSUSE<br />RHEL/CentOS<br />Fedora<br />Gentoo Linux<br />Debian<br />ArchLinux<br />FreeBSD<br />NetBSD<br />OpenBSD<br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /> | Amazon Web Services<br />Microsoft Azure<br />Google Cloud Platform<br />Oracle Cloud Infrastructure<br />Softlayer<br />Rackspace Public Cloud<br />IBM Cloud<br />Digital Ocean<br />Bigstep<br />Hetzner<br />Joyent<br />CloudSigma<br />Alibaba Cloud<br />OVH<br />OpenNebula<br />Exoscale<br />Scaleway<br />CloudStack<br />AltCloud<br />SmartOS<br />HyperOne<br />Rootbox<br /> | Bare metal installs<br />OpenStack<br />LXD<br />KVM<br />Metal-as-a-Service (MAAS)<br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br />|
+| Alpine Linux<br />ArchLinux<br />Debian<br />Fedora<br />FreeBSD<br />Gentoo Linux<br />NetBSD<br />OpenBSD<br />RHEL/CentOS<br />SLES/openSUSE<br />Ubuntu<br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /> | Amazon Web Services<br />Microsoft Azure<br />Google Cloud Platform<br />Oracle Cloud Infrastructure<br />Softlayer<br />Rackspace Public Cloud<br />IBM Cloud<br />Digital Ocean<br />Bigstep<br />Hetzner<br />Joyent<br />CloudSigma<br />Alibaba Cloud<br />OVH<br />OpenNebula<br />Exoscale<br />Scaleway<br />CloudStack<br />AltCloud<br />SmartOS<br />HyperOne<br />Rootbox<br /> | Bare metal installs<br />OpenStack<br />LXD<br />KVM<br />Metal-as-a-Service (MAAS)<br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br />|
 
 ## To start developing cloud-init
 
diff --git a/cloudinit/config/cc_apk_configure.py b/cloudinit/config/cc_apk_configure.py
new file mode 100644
index 00000000..84d7a0b6
--- /dev/null
+++ b/cloudinit/config/cc_apk_configure.py
@@ -0,0 +1,263 @@
+# Copyright (c) 2020 Dermot Bradley
+#
+# Author: Dermot Bradley <dermot_bradley@yahoo.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""Apk Configure: Configures apk repositories file."""
+
+from textwrap import dedent
+
+from cloudinit import log as logging
+from cloudinit import temp_utils
+from cloudinit import templater
+from cloudinit import util
+from cloudinit.config.schema import (
+    get_schema_doc, validate_cloudconfig_schema)
+from cloudinit.settings import PER_INSTANCE
+
+LOG = logging.getLogger(__name__)
+
+
+# If no mirror is specified then use this one
+DEFAULT_MIRROR = "https://alpine.global.ssl.fastly.net/alpine"
+
+
+REPOSITORIES_TEMPLATE = """\
+## template:jinja
+#
+# Created by cloud-init
+#
+# This file is written on first boot of an instance
+#
+
+{{ alpine_baseurl }}/{{ alpine_version }}/main
+{% if community_enabled -%}
+{{ alpine_baseurl }}/{{ alpine_version }}/community
+{% endif -%}
+{% if testing_enabled -%}
+{% if alpine_version != 'edge' %}
+#
+# Testing - using with non-Edge installation may cause problems!
+#
+{% endif %}
+{{ alpine_baseurl }}/edge/testing
+{% endif %}
+{% if local_repo != '' %}
+
+#
+# Local repo
+#
+{{ local_repo }}/{{ alpine_version }}
+{% endif %}
+
+"""
+
+
+frequency = PER_INSTANCE
+distros = ['alpine']
+schema = {
+    'id': 'cc_apk_configure',
+    'name': 'APK Configure',
+    'title': 'Configure apk repositories file',
+    'description': dedent("""\
+        This module handles configuration of the /etc/apk/repositories file.
+
+        .. note::
+          To ensure that apk configuration is valid yaml, any strings
+          containing special characters, especially ``:`` should be quoted.
+    """),
+    'distros': distros,
+    'examples': [
+        dedent("""\
+        # Keep the existing /etc/apk/repositories file unaltered.
+        apk_repos:
+            preserve_repositories: true
+        """),
+        dedent("""\
+        # Create repositories file for Alpine v3.12 main and community
+        # using default mirror site.
+        apk_repos:
+            alpine_repo:
+                community_enabled: true
+                version: 'v3.12'
+        """),
+        dedent("""\
+        # Create repositories file for Alpine Edge main, community, and
+        # testing using a specified mirror site and also a local repo.
+        apk_repos:
+            alpine_repo:
+                base_url: 'https://some-alpine-mirror/alpine'
+                community_enabled: true
+                testing_enabled: true
+                version: 'edge'
+            local_repo_base_url: 'https://my-local-server/local-alpine'
+        """),
+    ],
+    'frequency': frequency,
+    'type': 'object',
+    'properties': {
+        'apk_repos': {
+            'type': 'object',
+            'properties': {
+                'preserve_repositories': {
+                    'type': 'boolean',
+                    'default': False,
+                    'description': dedent("""\
+                        By default, cloud-init will generate a new repositories
+                        file ``/etc/apk/repositories`` based on any valid
+                        configuration settings specified within a apk_repos
+                        section of cloud config. To disable this behavior and
+                        preserve the repositories file from the pristine image,
+                        set ``preserve_repositories`` to ``true``.
+
+                        The ``preserve_repositories`` option overrides
+                        all other config keys that would alter
+                        ``/etc/apk/repositories``.
+                    """)
+                },
+                'alpine_repo': {
+                    'type': ['object', 'null'],
+                    'properties': {
+                        'base_url': {
+                            'type': 'string',
+                            'default': DEFAULT_MIRROR,
+                            'description': dedent("""\
+                                The base URL of an Alpine repository, or
+                                mirror, to download official packages from.
+                                If not specified then it defaults to ``{}``
+                            """.format(DEFAULT_MIRROR))
+                        },
+                        'community_enabled': {
+                            'type': 'boolean',
+                            'default': False,
+                            'description': dedent("""\
+                                Whether to add the Community repo to the
+                                repositories file. By default the Community
+                                repo is not included.
+                            """)
+                        },
+                        'testing_enabled': {
+                            'type': 'boolean',
+                            'default': False,
+                            'description': dedent("""\
+                                Whether to add the Testing repo to the
+                                repositories file. By default the Testing
+                                repo is not included. It is only recommended
+                                to use the Testing repo on a machine running
+                                the ``Edge`` version of Alpine as packages
+                                installed from Testing may have dependancies
+                                that conflict with those in non-Edge Main or
+                                Community repos."
+                            """)
+                        },
+                        'version': {
+                            'type': 'string',
+                            'description': dedent("""\
+                                The Alpine version to use (e.g. ``v3.12`` or
+                                ``edge``)
+                            """)
+                        },
+                    },
+                    'required': ['version'],
+                    'minProperties': 1,
+                    'additionalProperties': False,
+                },
+                'local_repo_base_url': {
+                    'type': 'string',
+                    'description': dedent("""\
+                        The base URL of an Alpine repository containing
+                        unofficial packages
+                    """)
+                }
+            },
+            'required': [],
+            'minProperties': 1,  # Either preserve_repositories or alpine_repo
+            'additionalProperties': False,
+        }
+    }
+}
+
+__doc__ = get_schema_doc(schema)
+
+
+def handle(name, cfg, cloud, log, _args):
+    """
+    Call to handle apk_repos sections in cloud-config file.
+
+    @param name: The module name "apk-configure" from cloud.cfg
+    @param cfg: A nested dict containing the entire cloud config contents.
+    @param cloud: The CloudInit object in use.
+    @param log: Pre-initialized Python logger object to use for logging.
+    @param _args: Any module arguments from cloud.cfg
+    """
+
+    # If there is no "apk_repos" section in the configuration
+    # then do nothing.
+    apk_section = cfg.get('apk_repos')
+    if not apk_section:
+        LOG.debug(("Skipping module named %s,"
+                   " no 'apk_repos' section found"), name)
+        return
+
+    validate_cloudconfig_schema(cfg, schema)
+
+    # If "preserve_repositories" is explicitly set to True in
+    # the configuration do nothing.
+    if util.get_cfg_option_bool(apk_section, 'preserve_repositories', False):
+        LOG.debug(("Skipping module named %s,"
+                   " 'preserve_repositories' is set"), name)
+        return
+
+    # If there is no "alpine_repo" subsection of "apk_repos" present in the
+    # configuration then do nothing, as at least "version" is required to
+    # create valid repositories entries.
+    alpine_repo = apk_section.get('alpine_repo')
+    if not alpine_repo:
+        LOG.debug(("Skipping module named %s,"
+                   " no 'alpine_repo' configuration found"), name)
+        return
+
+    # If there is no "version" value present in configuration then do nothing.
+    alpine_version = alpine_repo.get('version')
+    if not alpine_version:
+        LOG.debug(("Skipping module named %s,"
+                   " 'version' not specified in alpine_repo"), name)
+        return
+
+    local_repo = apk_section.get('local_repo_base_url', '')
+
+    _write_repositories_file(alpine_repo, alpine_version, local_repo)
+
+
+def _write_repositories_file(alpine_repo, alpine_version, local_repo):
+    """
+    Write the /etc/apk/repositories file with the specified entries.
+
+    @param alpine_repo: A nested dict of the alpine_repo configuration.
+    @param alpine_version: A string of the Alpine version to use.
+    @param local_repo: A string containing the base URL of a local repo.
+    """
+
+    repo_file = '/etc/apk/repositories'
+
+    alpine_baseurl = alpine_repo.get('base_url', DEFAULT_MIRROR)
+
+    params = {'alpine_baseurl': alpine_baseurl,
+              'alpine_version': alpine_version,
+              'community_enabled': alpine_repo.get('community_enabled'),
+              'testing_enabled': alpine_repo.get('testing_enabled'),
+              'local_repo': local_repo}
+
+    tfile = temp_utils.mkstemp(prefix='template_name-', suffix=".tmpl")
+    template_fn = tfile[1]  # Filepath is second item in tuple
+    util.write_file(template_fn, content=REPOSITORIES_TEMPLATE)
+
+    LOG.debug('Generating Alpine repository configuration file: %s',
+              repo_file)
+    templater.render_to_file(template_fn, repo_file, params)
+    # Clean up temporary template
+    util.del_file(template_fn)
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py
index 910b78de..3c453d91 100644
--- a/cloudinit/config/cc_ca_certs.py
+++ b/cloudinit/config/cc_ca_certs.py
@@ -16,11 +16,16 @@ can be removed from the system with the configuration option
     certificates must be specified using valid yaml. in order to specify a
     multiline certificate, the yaml multiline list syntax must be used
 
+.. note::
+    For Alpine Linux the "remove-defaults" functionality works if the
+    ca-certificates package is installed but not if the
+    ca-certificates-bundle package is installed.
+
 **Internal name:** ``cc_ca_certs``
 
 **Module frequency:** per instance
 
-**Supported distros:** ubuntu, debian
+**Supported distros:** alpine, debian, ubuntu
 
 **Config keys**::
 
@@ -45,7 +50,7 @@ CA_CERT_CONFIG = "/etc/ca-certificates.conf"
 CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/"
 CA_CERT_FULL_PATH = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME)
 
-distros = ['ubuntu', 'debian']
+distros = ['alpine', 'debian', 'ubuntu']
 
 
 def update_ca_certs():
@@ -83,7 +88,7 @@ def add_ca_certs(certs):
         util.write_file(CA_CERT_CONFIG, out, omode="wb")
 
 
-def remove_default_ca_certs():
+def remove_default_ca_certs(distro_name):
     """
     Removes all default trusted CA certificates from the system. To actually
     apply the change you must also call L{update_ca_certs}.
@@ -91,11 +96,14 @@ def remove_default_ca_certs():
     util.delete_dir_contents(CA_CERT_PATH)
     util.delete_dir_contents(CA_CERT_SYSTEM_PATH)
     util.write_file(CA_CERT_CONFIG, "", mode=0o644)
-    debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no"
-    subp.subp(('debconf-set-selections', '-'), debconf_sel)
+
+    if distro_name != 'alpine':
+        debconf_sel = (
+            "ca-certificates ca-certificates/trust_new_crts " + "select no")
+        subp.subp(('debconf-set-selections', '-'), debconf_sel)
 
 
-def handle(name, cfg, _cloud, log, _args):
+def handle(name, cfg, cloud, log, _args):
     """
     Call to handle ca-cert sections in cloud-config file.
 
@@ -117,7 +125,7 @@ def handle(name, cfg, _cloud, log, _args):
     # default trusted CA certs first.
     if ca_cert_cfg.get("remove-defaults", False):
         log.debug("Removing default certificates")
-        remove_default_ca_certs()
+        remove_default_ca_certs(cloud.distro.name)
 
     # If we are given any new trusted CA certs to add, add them.
     if "trusted" in ca_cert_cfg:
diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py
index 7d3f73ff..3d7279d6 100644
--- a/cloudinit/config/cc_ntp.py
+++ b/cloudinit/config/cc_ntp.py
@@ -24,7 +24,8 @@ LOG = logging.getLogger(__name__)
 frequency = PER_INSTANCE
 NTP_CONF = '/etc/ntp.conf'
 NR_POOL_SERVERS = 4
-distros = ['centos', 'debian', 'fedora', 'opensuse', 'rhel', 'sles', 'ubuntu']
+distros = ['alpine', 'centos', 'debian', 'fedora', 'opensuse', 'rhel',
+           'sles', 'ubuntu']
 
 NTP_CLIENT_CONFIG = {
     'chrony': {
@@ -63,6 +64,17 @@ NTP_CLIENT_CONFIG = {
 
 # This is Distro-specific configuration overrides of the base config
 DISTRO_CLIENT_CONFIG = {
+    'alpine': {
+        'chrony': {
+            'confpath': '/etc/chrony/chrony.conf',
+            'service_name': 'chronyd',
+        },
+        'ntp': {
+            'confpath': '/etc/ntp.conf',
+            'packages': [],
+            'service_name': 'ntpd',
+        },
+    },
     'debian': {
         'chrony': {
             'confpath': '/etc/chrony/chrony.conf',
@@ -114,11 +126,11 @@ schema = {
         Handle ntp configuration. If ntp is not installed on the system and
         ntp configuration is specified, ntp will be installed. If there is a
         default ntp config file in the image or one is present in the
-        distro's ntp package, it will be copied to ``/etc/ntp.conf.dist``
-        before any changes are made. A list of ntp pools and ntp servers can
-        be provided under the ``ntp`` config key. If no ntp ``servers`` or
-        ``pools`` are provided, 4 pools will be used in the format
-        ``{0-3}.{distro}.pool.ntp.org``."""),
+        distro's ntp package, it will be copied to a file with ``.dist``
+        appended to the filename before any changes are made. A list of ntp
+        pools and ntp servers can be provided under the ``ntp`` config key.
+        If no ntp ``servers`` or ``pools`` are provided, 4 pools will be used
+        in the format ``{0-3}.{distro}.pool.ntp.org``."""),
     'distros': distros,
     'examples': [
         dedent("""\
@@ -171,7 +183,10 @@ schema = {
                     'description': dedent("""\
                         List of ntp pools. If both pools and servers are
                         empty, 4 default pool servers will be provided of
-                        the format ``{0-3}.{distro}.pool.ntp.org``.""")
+                        the format ``{0-3}.{distro}.pool.ntp.org``. NOTE:
+                        for Alpine Linux when using the Busybox NTP client
+                        this setting will be ignored due to the limited
+                        functionality of Busybox's ntpd.""")
                 },
                 'servers': {
                     'type': 'array',
@@ -364,21 +379,30 @@ def generate_server_names(distro):
     """
     names = []
     pool_distro = distro
-    # For legal reasons x.pool.sles.ntp.org does not exist,
-    # use the opensuse pool
+
     if distro == 'sles':
+        # For legal reasons x.pool.sles.ntp.org does not exist,
+        # use the opensuse pool
         pool_distro = 'opensuse'
+    elif distro == 'alpine':
+        # Alpine-specific pool (i.e. x.alpine.pool.ntp.org) does not exist
+        # so use general x.pool.ntp.org instead.
+        pool_distro = ''
+
     for x in range(0, NR_POOL_SERVERS):
-        name = "%d.%s.pool.ntp.org" % (x, pool_distro)
-        names.append(name)
+        names.append(".".join(
+            [n for n in [str(x)] + [pool_distro] + ['pool.ntp.org'] if n]))
+
     return names
 
 
-def write_ntp_config_template(distro_name, servers=None, pools=None,
-                              path=None, template_fn=None, template=None):
+def write_ntp_config_template(distro_name, service_name=None, servers=None,
+                              pools=None, path=None, template_fn=None,
+                              template=None):
     """Render a ntp client configuration for the specified client.
 
     @param distro_name: string.  The distro class name.
+    @param service_name: string. The name of the NTP client service.
     @param servers: A list of strings specifying ntp servers. Defaults to empty
     list.
     @param pools: A list of strings specifying ntp pools. Defaults to empty
@@ -397,7 +421,14 @@ def write_ntp_config_template(distro_name, servers=None, pools=None,
     if not pools:
         pools = []
 
-    if len(servers) == 0 and len(pools) == 0:
+    if (len(servers) == 0 and distro_name == 'alpine' and
+            service_name == 'ntpd'):
+        # Alpine's Busybox ntpd only understands "servers" configuration
+        # and not "pool" configuration.
+        servers = generate_server_names(distro_name)
+        LOG.debug(
+            'Adding distro default ntp servers: %s', ','.join(servers))
+    elif len(servers) == 0 and len(pools) == 0:
         pools = generate_server_names(distro_name)
         LOG.debug(
             'Adding distro default ntp pool servers: %s', ','.join(pools))
@@ -532,6 +563,8 @@ def handle(name, cfg, cloud, log, _args):
             raise RuntimeError(msg)
 
     write_ntp_config_template(cloud.distro.name,
+                              service_name=ntp_client_config.get(
+                                  'service_name'),
                               servers=ntp_cfg.get('servers', []),
                               pools=ntp_cfg.get('pools', []),
                               path=ntp_client_config.get('confpath'),
diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py
index 41ffb46c..ab953a0d 100644
--- a/cloudinit/config/cc_power_state_change.py
+++ b/cloudinit/config/cc_power_state_change.py
@@ -22,9 +22,8 @@ The ``delay`` key specifies a duration to be added onto any shutdown command
 used. Therefore, if a 5 minute delay and a 120 second shutdown are specified,
 the maximum amount of time between cloud-init starting and the system shutting
 down is 7 minutes, and the minimum amount of time is 5 minutes. The ``delay``
-key must have an argument in a form that the ``shutdown`` utility recognizes.
-The most common format is the form ``+5`` for 5 minutes. See ``man shutdown``
-for more options.
+key must have an argument in either the form ``+5`` for 5 minutes or ``now``
+for immediate shutdown.
 
 Optionally, a command can be run to determine whether or not
 the system should shut down. The command to be run should be specified in the
@@ -33,6 +32,10 @@ the system should shut down. The command to be run should be specified in the
 ``condition`` key is omitted or the command specified by the ``condition``
 key returns 0.
 
+.. note::
+    With Alpine Linux any message value specified is ignored as Alpine's halt,
+    poweroff, and reboot commands do not support broadcasting a message.
+
 **Internal name:** ``cc_power_state_change``
 
 **Module frequency:** per instance
@@ -112,9 +115,9 @@ def check_condition(cond, log=None):
         return False
 
 
-def handle(_name, cfg, _cloud, log, _args):
+def handle(_name, cfg, cloud, log, _args):
     try:
-        (args, timeout, condition) = load_power_state(cfg)
+        (args, timeout, condition) = load_power_state(cfg, cloud.distro.name)
         if args is None:
             log.debug("no power_state provided. doing nothing")
             return
@@ -141,7 +144,19 @@ def handle(_name, cfg, _cloud, log, _args):
                  condition, execmd, [args, devnull_fp])
 
 
-def load_power_state(cfg):
+def convert_delay(delay, fmt=None, scale=None):
+    if not fmt:
+        fmt = "+%s"
+    if not scale:
+        scale = 1
+
+    if delay != "now":
+        delay = fmt % int(int(delay) * int(scale))
+
+    return delay
+
+
+def load_power_state(cfg, distro_name):
     # returns a tuple of shutdown_command, timeout
     # shutdown_command is None if no config found
     pstate = cfg.get('power_state')
@@ -161,20 +176,34 @@ def load_power_state(cfg):
             (','.join(opt_map.keys()), mode))
 
     delay = pstate.get("delay", "now")
-    # convert integer 30 or string '30' to '+30'
+    message = pstate.get("message")
+    scale = 1
+    fmt = "+%s"
+    command = ["shutdown", opt_map[mode]]
+
+    if distro_name == 'alpine':
+        # Convert integer 30 or string '30' to '1800' (seconds) as Alpine's
+        # halt/poweroff/reboot commands take seconds rather than minutes.
+        scale = 60
+        # No "+" in front of delay value as not supported by Alpine's commands.
+        fmt = "%s"
+        if delay == "now":
+            # Alpine's commands do not understand "now".
+            delay = "0"
+        command = [mode, "-d"]
+        # Alpine's commands don't support a message.
+        message = None
+
     try:
-        delay = "+%s" % int(delay)
+        delay = convert_delay(delay, fmt=fmt, scale=scale)
     except ValueError:
-        pass
-
-    if delay != "now" and not re.match(r"\+[0-9]+", delay):
         raise TypeError(
             "power_state[delay] must be 'now' or '+m' (minutes)."
             " found '%s'." % delay)
 
-    args = ["shutdown", opt_map[mode], delay]
-    if pstate.get("message"):
-        args.append(pstate.get("message"))
+    args = command + [delay]
+    if message:
+        args.append(message)
 
     try:
         timeout = float(pstate.get('timeout', 30.0))
diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py
index 69f4768a..519e66eb 100644
--- a/cloudinit/config/cc_resolv_conf.py
+++ b/cloudinit/config/cc_resolv_conf.py
@@ -30,7 +30,7 @@ are configured correctly.
 
 **Module frequency:** per instance
 
-**Supported distros:** fedora, rhel, sles
+**Supported distros:** alpine, fedora, rhel, sles
 
 **Config keys**::
 
@@ -55,7 +55,7 @@ LOG = logging.getLogger(__name__)
 
 frequency = PER_INSTANCE
 
-distros = ['fedora', 'opensuse', 'rhel', 'sles']
+distros = ['alpine', 'fedora', 'opensuse', 'rhel', 'sles']
 
 
 def generate_resolv_conf(template_fn, params, target_fname="/etc/resolv.conf"):
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index c7163e1c..effb4276 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -40,12 +40,13 @@ from .networking import LinuxNetworking
 ALL_DISTROS = 'all'
 
 OSFAMILIES = {
+    'alpine': ['alpine'],
+    'arch': ['arch'],
     'debian': ['debian', 'ubuntu'],
-    'redhat': ['amazon', 'centos', 'fedora', 'rhel'],
-    'gentoo': ['gentoo'],
     'freebsd': ['freebsd'],
+    'gentoo': ['gentoo'],
+    'redhat': ['amazon', 'centos', 'fedora', 'rhel'],
     'suse': ['opensuse', 'sles'],
-    'arch': ['arch'],
 }
 
 LOG = logging.getLogger(__name__)
diff --git a/cloudinit/distros/alpine.py b/cloudinit/distros/alpine.py
new file mode 100644
index 00000000..e42443fc
--- /dev/null
+++ b/cloudinit/distros/alpine.py
@@ -0,0 +1,165 @@
+# Copyright (C) 2016 Matt Dainty
+# Copyright (C) 2020 Dermot Bradley
+#
+# Author: Matt Dainty <matt@bodgit-n-scarper.com>
+# Author: Dermot Bradley <dermot_bradley@yahoo.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit import distros
+from cloudinit import helpers
+from cloudinit import log as logging
+from cloudinit import subp
+from cloudinit import util
+
+from cloudinit.distros.parsers.hostname import HostnameConf
+
+from cloudinit.settings import PER_INSTANCE
+
+LOG = logging.getLogger(__name__)
+
+NETWORK_FILE_HEADER = """\
+# This file is generated from information provided by the datasource. Changes
+# to it will not persist across an instance reboot. To disable cloud-init's
+# network configuration capabilities, write a file
+# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following:
+# network: {config: disabled}
+
+"""
+
+
+class Distro(distros.Distro):
+    init_cmd = ['rc-service']  # init scripts
+    locale_conf_fn = "/etc/profile.d/locale.sh"
+    network_conf_fn = "/etc/network/interfaces"
+    renderer_configs = {
+        "eni": {"eni_path": network_conf_fn,
+                "eni_header": NETWORK_FILE_HEADER}
+    }
+
+    def __init__(self, name, cfg, paths):
+        distros.Distro.__init__(self, name, cfg, paths)
+        # This will be used to restrict certain
+        # calls from repeatly happening (when they
+        # should only happen say once per instance...)
+        self._runner = helpers.Runners(paths)
+        self.default_locale = 'C.UTF-8'
+        self.osfamily = 'alpine'
+        cfg['ssh_svcname'] = 'sshd'
+
+    def get_locale(self):
+        """The default locale for Alpine Linux is different than
+           cloud-init's DataSource default.
+        """
+        return self.default_locale
+
+    def apply_locale(self, locale, out_fn=None):
+        # Alpine has limited locale support due to musl library limitations
+
+        if not locale:
+            locale = self.default_locale
+        if not out_fn:
+            out_fn = self.locale_conf_fn
+
+        lines = [
+            "#",
+            "# This file is created by cloud-init once per new instance boot",
+            "#",
+            "export CHARSET=UTF-8",
+            "export LANG=%s" % locale,
+            "export LC_COLLATE=C",
+            "",
+        ]
+        util.write_file(out_fn, "\n".join(lines), 0o644)
+
+    def install_packages(self, pkglist):
+        self.update_package_sources()
+        self.package_command('add', pkgs=pkglist)
+
+    def _write_network_config(self, netconfig):
+        return self._supported_write_network_config(netconfig)
+
+    def _bring_up_interfaces(self, device_names):
+        use_all = False
+        for d in device_names:
+            if d == 'all':
+                use_all = True
+        if use_all:
+            return distros.Distro._bring_up_interface(self, '-a')
+        else:
+            return distros.Distro._bring_up_interfaces(self, device_names)
+
+    def _write_hostname(self, your_hostname, out_fn):
+        conf = None
+        try:
+            # Try to update the previous one
+            # so lets see if we can read it first.
+            conf = self._read_hostname_conf(out_fn)
+        except IOError:
+            pass
+        if not conf:
+            conf = HostnameConf('')
+        conf.set_hostname(your_hostname)
+        util.write_file(out_fn, str(conf), 0o644)
+
+    def _read_system_hostname(self):
+        sys_hostname = self._read_hostname(self.hostname_conf_fn)
+        return (self.hostname_conf_fn, sys_hostname)
+
+    def _read_hostname_conf(self, filename):
+        conf = HostnameConf(util.load_file(filename))
+        conf.parse()
+        return conf
+
+    def _read_hostname(self, filename, default=None):
+        hostname = None
+        try:
+            conf = self._read_hostname_conf(filename)
+            hostname = conf.hostname
+        except IOError:
+            pass
+        if not hostname:
+            return default
+        return hostname
+
+    def _get_localhost_ip(self):
+        return "127.0.1.1"
+
+    def set_timezone(self, tz):
+        distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz))
+
+    def package_command(self, command, args=None, pkgs=None):
+        if pkgs is None:
+            pkgs = []
+
+        cmd = ['apk']
+        # Redirect output
+        cmd.append("--quiet")
+
+        if args and isinstance(args, str):
+            cmd.append(args)
+        elif args and isinstance(args, list):
+            cmd.extend(args)
+
+        if command:
+            cmd.append(command)
+
+        pkglist = util.expand_package_list('%s-%s', pkgs)
+        cmd.extend(pkglist)
+
+        # Allow the output of this to flow outwards (ie not be captured)
+        subp.subp(cmd, capture=False)
+
+    def update_package_sources(self):
+        self._runner.run("update-sources", self.package_command,
+                         ["update"], freq=PER_INSTANCE)
+
+    @property
+    def preferred_ntp_clients(self):
+        """Allow distro to determine the preferred ntp client list"""
+        if not self._preferred_ntp_clients:
+            self._preferred_ntp_clients = ['chrony', 'ntp']
+
+        return self._preferred_ntp_clients
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/util.py b/cloudinit/util.py
index edd37039..dd263803 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -548,7 +548,8 @@ def system_info():
     if system == "linux":
         linux_dist = info['dist'][0].lower()
         if linux_dist in (
-                'arch', 'centos', 'debian', 'fedora', 'rhel', 'suse'):
+                'alpine', 'arch', 'centos', 'debian', 'fedora', 'rhel',
+                'suse'):
             var = linux_dist
         elif linux_dist in ('ubuntu', 'linuxmint', 'mint'):
             var = 'ubuntu'
diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl
index b44cbce7..2beb9b0c 100644
--- a/config/cloud.cfg.tmpl
+++ b/config/cloud.cfg.tmpl
@@ -21,7 +21,7 @@ disable_root: false
 disable_root: true
 {% endif %}
 
-{% if variant in ["amazon", "centos", "fedora", "rhel"] %}
+{% if variant in ["alpine", "amazon", "centos", "fedora", "rhel"] %}
 mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2']
 {% if variant == "amazon" %}
 resize_rootfs: noblock
@@ -71,6 +71,9 @@ cloud_init_modules:
  - set_hostname
  - update_hostname
  - update_etc_hosts
+{% if variant in ["alpine"] %}
+ - resolv_conf
+{% endif %}
 {% if not variant.endswith("bsd") %}
  - ca-certs
  - rsyslog
@@ -104,6 +107,9 @@ cloud_config_modules:
 {% if variant in ["suse"] %}
  - zypper-add-repo
 {% endif %}
+{% if variant in ["alpine"] %}
+ - apk-configure
+{% endif %}
 {% if variant not in ["freebsd", "netbsd"] %}
  - ntp
 {% endif %}
@@ -145,7 +151,9 @@ cloud_final_modules:
 # (not accessible to handlers/transforms)
 system_info:
    # This will affect which distro class gets used
-{% if variant in ["amazon", "arch", "centos", "debian", "fedora", "freebsd", "netbsd", "openbsd", "rhel", "suse", "ubuntu"] %}
+{% if variant in ["alpine", "amazon", "arch", "centos", "debian",
+                  "fedora", "freebsd", "netbsd", "openbsd", "rhel",
+                  "suse", "ubuntu"] %}
    distro: {{ variant }}
 {% else %}
    # Unknown/fallback distro.
@@ -196,7 +204,8 @@ system_info:
          primary: http://ports.ubuntu.com/ubuntu-ports
          security: http://ports.ubuntu.com/ubuntu-ports
    ssh_svcname: ssh
-{% elif variant in ["amazon", "arch", "centos", "fedora", "rhel", "suse"] %}
+{% elif variant in ["alpine", "amazon", "arch", "centos", "fedora",
+                    "rhel", "suse"] %}
    # Default user name + that default users groups (if added/used)
    default_user:
 {% if variant == "amazon" %}
@@ -210,13 +219,19 @@ system_info:
 {% endif %}
 {% if variant == "suse" %}
      groups: [cdrom, users]
+{% elif variant == "alpine" %}
+     groups: [adm, sudo]
 {% elif variant == "arch" %}
      groups: [wheel, users]
 {% else %}
      groups: [wheel, adm, systemd-journal]
 {% endif %}
      sudo: ["ALL=(ALL) NOPASSWD:ALL"]
+{% if variant == "alpine" %}
+     shell: /bin/ash
+{% else %}
      shell: /bin/bash
+{% endif %}
    # Other config here will be given to the distro class and/or path classes
    paths:
       cloud_dir: /var/lib/cloud/
diff --git a/doc/rtd/topics/availability.rst b/doc/rtd/topics/availability.rst
index 84490460..8f56a7d2 100644
--- a/doc/rtd/topics/availability.rst
+++ b/doc/rtd/topics/availability.rst
@@ -17,16 +17,17 @@ Distributions
 Cloud-init has support across all major Linux distributions, FreeBSD, NetBSD
 and OpenBSD:
 
-- Ubuntu
-- SLES/openSUSE
-- RHEL/CentOS
-- Fedora
-- Gentoo Linux
-- Debian
+- Alpine Linux
 - ArchLinux
+- Debian
+- Fedora
 - FreeBSD
+- Gentoo Linux
 - NetBSD
 - OpenBSD
+- RHEL/CentOS
+- SLES/openSUSE
+- Ubuntu
 
 Clouds
 ======
diff --git a/doc/rtd/topics/instancedata.rst b/doc/rtd/topics/instancedata.rst
index 845098bb..255245a4 100644
--- a/doc/rtd/topics/instancedata.rst
+++ b/doc/rtd/topics/instancedata.rst
@@ -132,6 +132,7 @@ This shall be the distro name, version and release as determined by
 
 Example output:
 
+- alpine, 3.12.0, ''
 - centos, 7.5, core
 - debian, 9, stretch
 - freebsd, 12.0-release-p10,
diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst
index 9c9be804..e30fe0fe 100644
--- a/doc/rtd/topics/modules.rst
+++ b/doc/rtd/topics/modules.rst
@@ -6,6 +6,7 @@ Modules
 *******
 .. contents:: Table of Contents
 
+.. automodule:: cloudinit.config.cc_apk_configure
 .. automodule:: cloudinit.config.cc_apt_configure
 .. automodule:: cloudinit.config.cc_apt_pipelining
 .. automodule:: cloudinit.config.cc_bootcmd
diff --git a/doc/rtd/topics/network-config.rst b/doc/rtd/topics/network-config.rst
index 8eeadebf..08db04d8 100644
--- a/doc/rtd/topics/network-config.rst
+++ b/doc/rtd/topics/network-config.rst
@@ -165,7 +165,7 @@ supported formats.  The following ``renderers`` are supported in cloud-init:
 - **ENI**
 
 /etc/network/interfaces or ``ENI`` is supported by the ``ifupdown`` package
-found in Ubuntu and Debian.
+found in Alpine Linux, Debian and Ubuntu.
 
 - **Netplan**
 
diff --git a/templates/chrony.conf.alpine.tmpl b/templates/chrony.conf.alpine.tmpl
new file mode 100644
index 00000000..45efc18c
--- /dev/null
+++ b/templates/chrony.conf.alpine.tmpl
@@ -0,0 +1,38 @@
+## template:jinja
+# Welcome to the chrony configuration file. See chrony.conf(5) for more
+# information about usable directives.
+{% if pools %}# pools
+{% endif %}
+{% for pool in pools -%}
+pool {{pool}} iburst
+{% endfor %}
+{%- if servers %}# servers
+{% endif %}
+{% for server in servers -%}
+server {{server}} iburst
+{% endfor %}
+
+# This directive specifies the location of the file containing ID/key pairs for
+# NTP authentication.
+keyfile /etc/chrony/chrony.keys
+
+# This directive specifies the file into which chronyd will store the rate
+# information.
+driftfile /var/lib/chrony/chrony.drift
+
+# Uncomment the following line to turn logging on.
+#log tracking measurements statistics
+
+# Log files location.
+logdir /var/log/chrony
+
+# Stop bad estimates upsetting machine clock.
+maxupdateskew 100.0
+
+# This directive enables kernel synchronisation (every 11 minutes) of the
+# real-time clock. Note that it can’t be used along with the 'rtcfile' directive.
+rtcsync
+
+# Step the system clock instead of slewing it if the adjustment is larger than
+# one second, but only in the first three clock updates.
+makestep 1 3
diff --git a/templates/hosts.alpine.tmpl b/templates/hosts.alpine.tmpl
new file mode 100644
index 00000000..33c1a941
--- /dev/null
+++ b/templates/hosts.alpine.tmpl
@@ -0,0 +1,28 @@
+## template:jinja
+{#
+This file /etc/cloud/templates/hosts.alpine.tmpl is only utilized
+if enabled in cloud-config.  Specifically, in order to enable it
+you need to add the following to config:
+  manage_etc_hosts: True
+-#}
+# Your system has configured 'manage_etc_hosts' as True.
+# As a result, if you wish for changes to this file to persist
+# then you will need to either
+# a.) make changes to the master file in /etc/cloud/templates/hosts.alpine.tmpl
+# b.) change or remove the value of 'manage_etc_hosts' in
+#     /etc/cloud/cloud.cfg or cloud-config from user-data
+#
+# The following lines are desirable for IPv4 capable hosts
+127.0.1.1 {{fqdn}} {{hostname}}
+127.0.0.1 localhost.localdomain localhost
+127.0.0.1 localhost4.localdomain4 localhost4
+
+# The following lines are desirable for IPv6 capable hosts
+::1 {{fqdn}} {{hostname}}
+::1 localhost6.localdomain6 localhost6
+
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+ff02::3 ip6-allhosts
diff --git a/templates/ntp.conf.alpine.tmpl b/templates/ntp.conf.alpine.tmpl
new file mode 100644
index 00000000..59ca8fc1
--- /dev/null
+++ b/templates/ntp.conf.alpine.tmpl
@@ -0,0 +1,10 @@
+## template:jinja
+# /etc/ntp.conf
+#
+# Configuration for Busybox ntpd - it only supports "server" lines.
+
+{% if servers %}# Servers
+{% endif %}
+{% for server in servers -%}
+server {{server}}
+{% endfor %}
diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py
index 43d996b9..dcf0fe5a 100644
--- a/tests/unittests/test_cli.py
+++ b/tests/unittests/test_cli.py
@@ -224,7 +224,7 @@ class TestCLI(test_helpers.FilesystemMockingTestCase):
         self._call_main(['cloud-init', 'devel', 'schema', '--docs', 'all'])
         expected_doc_sections = [
             '**Supported distros:** all',
-            '**Supported distros:** centos, debian, fedora',
+            '**Supported distros:** alpine, centos, debian, fedora',
             '**Config schema**:\n    **resize_rootfs:** (true/false/noblock)',
             '**Examples**::\n\n    runcmd:\n        - [ ls, -l, / ]\n'
         ]
diff --git a/tests/unittests/test_handler/test_handler_apk_configure.py b/tests/unittests/test_handler/test_handler_apk_configure.py
new file mode 100644
index 00000000..8acc0b33
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_apk_configure.py
@@ -0,0 +1,299 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+""" test_apk_configure
+Test creation of repositories file
+"""
+
+import logging
+import os
+import textwrap
+
+from cloudinit import (cloud, helpers, util)
+
+from cloudinit.config import cc_apk_configure
+from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock)
+
+REPO_FILE = "/etc/apk/repositories"
+DEFAULT_MIRROR_URL = "https://alpine.global.ssl.fastly.net/alpine"
+CC_APK = 'cloudinit.config.cc_apk_configure'
+
+
+class TestNoConfig(FilesystemMockingTestCase):
+    def setUp(self):
+        super(TestNoConfig, self).setUp()
+        self.add_patch(CC_APK + '._write_repositories_file', 'm_write_repos')
+        self.name = "apk-configure"
+        self.cloud_init = None
+        self.log = logging.getLogger("TestNoConfig")
+        self.args = []
+
+    def test_no_config(self):
+        """
+        Test that nothing is done if no apk-configure
+        configuration is provided.
+        """
+        config = util.get_builtin_cfg()
+
+        cc_apk_configure.handle(self.name, config, self.cloud_init,
+                                self.log, self.args)
+
+        self.assertEqual(0, self.m_write_repos.call_count)
+
+
+class TestConfig(FilesystemMockingTestCase):
+    def setUp(self):
+        super(TestConfig, self).setUp()
+        self.new_root = self.tmp_dir()
+        self.new_root = self.reRoot(root=self.new_root)
+        for dirname in ['tmp', 'etc/apk']:
+            util.ensure_dir(os.path.join(self.new_root, dirname))
+        self.paths = helpers.Paths({'templates_dir': self.new_root})
+        self.name = "apk-configure"
+        self.cloud = cloud.Cloud(None, self.paths, None, None, None)
+        self.log = logging.getLogger("TestNoConfig")
+        self.args = []
+
+    @mock.patch(CC_APK + '._write_repositories_file')
+    def test_no_repo_settings(self, m_write_repos):
+        """
+        Test that nothing is written if the 'alpine-repo' key
+        is not present.
+        """
+        config = {"apk_repos": {}}
+
+        cc_apk_configure.handle(self.name, config, self.cloud, self.log,
+                                self.args)
+
+        self.assertEqual(0, m_write_repos.call_count)
+
+    @mock.patch(CC_APK + '._write_repositories_file')
+    def test_empty_repo_settings(self, m_write_repos):
+        """
+        Test that nothing is written if 'alpine_repo' list is empty.
+        """
+        config = {"apk_repos": {"alpine_repo": []}}
+
+        cc_apk_configure.handle(self.name, config, self.cloud, self.log,
+                                self.args)
+
+        self.assertEqual(0, m_write_repos.call_count)
+
+    def test_only_main_repo(self):
+        """
+        Test when only details of main repo is written to file.
+        """
+        alpine_version = 'v3.12'
+        config = {
+            "apk_repos": {
+                "alpine_repo": {
+                    "version": alpine_version
+                }
+            }
+        }
+
+        cc_apk_configure.handle(self.name, config, self.cloud, self.log,
+                                self.args)
+
+        expected_content = textwrap.dedent("""\
+            #
+            # Created by cloud-init
+            #
+            # This file is written on first boot of an instance
+            #
+
+            {0}/{1}/main
+
+            """.format(DEFAULT_MIRROR_URL, alpine_version))
+
+        self.assertEqual(expected_content, util.load_file(REPO_FILE))
+
+    def test_main_and_community_repos(self):
+        """
+        Test when only details of main and community repos are
+        written to file.
+        """
+        alpine_version = 'edge'
+        config = {
+            "apk_repos": {
+                "alpine_repo": {
+                    "version": alpine_version,
+                    "community_enabled": True
+                }
+            }
+        }
+
+        cc_apk_configure.handle(self.name, config, self.cloud, self.log,
+                                self.args)
+
+        expected_content = textwrap.dedent("""\
+            #
+            # Created by cloud-init
+            #
+            # This file is written on first boot of an instance
+            #
+
+            {0}/{1}/main
+            {0}/{1}/community
+
+            """.format(DEFAULT_MIRROR_URL, alpine_version))
+
+        self.assertEqual(expected_content, util.load_file(REPO_FILE))
+
+    def test_main_community_testing_repos(self):
+        """
+        Test when details of main, community and testing repos
+        are written to file.
+        """
+        alpine_version = 'v3.12'
+        config = {
+            "apk_repos": {
+                "alpine_repo": {
+                    "version": alpine_version,
+                    "community_enabled": True,
+                    "testing_enabled": True
+                }
+            }
+        }
+
+        cc_apk_configure.handle(self.name, config, self.cloud, self.log,
+                                self.args)
+
+        expected_content = textwrap.dedent("""\
+            #
+            # Created by cloud-init
+            #
+            # This file is written on first boot of an instance
+            #
+
+            {0}/{1}/main
+            {0}/{1}/community
+            #
+            # Testing - using with non-Edge installation may cause problems!
+            #
+            {0}/edge/testing
+
+            """.format(DEFAULT_MIRROR_URL, alpine_version))
+
+        self.assertEqual(expected_content, util.load_file(REPO_FILE))
+
+    def test_edge_main_community_testing_repos(self):
+        """
+        Test when details of main, community and testing repos
+        for Edge version of Alpine are written to file.
+        """
+        alpine_version = 'edge'
+        config = {
+            "apk_repos": {
+                "alpine_repo": {
+                    "version": alpine_version,
+                    "community_enabled": True,
+                    "testing_enabled": True
+                }
+            }
+        }
+
+        cc_apk_configure.handle(self.name, config, self.cloud, self.log,
+                                self.args)
+
+        expected_content = textwrap.dedent("""\
+            #
+            # Created by cloud-init
+            #
+            # This file is written on first boot of an instance
+            #
+
+            {0}/{1}/main
+            {0}/{1}/community
+            {0}/{1}/testing
+
+            """.format(DEFAULT_MIRROR_URL, alpine_version))
+
+        self.assertEqual(expected_content, util.load_file(REPO_FILE))
+
+    def test_main_community_testing_local_repos(self):
+        """
+        Test when details of main, community, testing and
+        local repos are written to file.
+        """
+        alpine_version = 'v3.12'
+        local_repo_url = 'http://some.mirror/whereever'
+        config = {
+            "apk_repos": {
+                "alpine_repo": {
+                    "version": alpine_version,
+                    "community_enabled": True,
+                    "testing_enabled": True
+                },
+                "local_repo_base_url": local_repo_url
+            }
+        }
+
+        cc_apk_configure.handle(self.name, config, self.cloud, self.log,
+                                self.args)
+
+        expected_content = textwrap.dedent("""\
+            #
+            # Created by cloud-init
+            #
+            # This file is written on first boot of an instance
+            #
+
+            {0}/{1}/main
+            {0}/{1}/community
+            #
+            # Testing - using with non-Edge installation may cause problems!
+            #
+            {0}/edge/testing
+
+            #
+            # Local repo
+            #
+            {2}/{1}
+
+            """.format(DEFAULT_MIRROR_URL, alpine_version, local_repo_url))
+
+        self.assertEqual(expected_content, util.load_file(REPO_FILE))
+
+    def test_edge_main_community_testing_local_repos(self):
+        """
+        Test when details of main, community, testing and local repos
+        for Edge version of Alpine are written to file.
+        """
+        alpine_version = 'edge'
+        local_repo_url = 'http://some.mirror/whereever'
+        config = {
+            "apk_repos": {
+                "alpine_repo": {
+                    "version": alpine_version,
+                    "community_enabled": True,
+                    "testing_enabled": True
+                },
+                "local_repo_base_url": local_repo_url
+            }
+        }
+
+        cc_apk_configure.handle(self.name, config, self.cloud, self.log,
+                                self.args)
+
+        expected_content = textwrap.dedent("""\
+            #
+            # Created by cloud-init
+            #
+            # This file is written on first boot of an instance
+            #
+
+            {0}/{1}/main
+            {0}/{1}/community
+            {0}/edge/testing
+
+            #
+            # Local repo
+            #
+            {2}/{1}
+
+            """.format(DEFAULT_MIRROR_URL, alpine_version, local_repo_url))
+
+        self.assertEqual(expected_content, util.load_file(REPO_FILE))
+
+
+# vi: ts=4 expandtab
diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py
index c1aff181..e74a0a08 100644
--- a/tests/unittests/test_handler/test_handler_ca_certs.py
+++ b/tests/unittests/test_handler/test_handler_ca_certs.py
@@ -1,6 +1,7 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit import cloud
+from cloudinit import distros
 from cloudinit.config import cc_ca_certs
 from cloudinit import helpers
 from cloudinit import subp
@@ -46,8 +47,9 @@ class TestConfig(TestCase):
     def setUp(self):
         super(TestConfig, self).setUp()
         self.name = "ca-certs"
+        distro = self._fetch_distro('ubuntu')
         self.paths = None
-        self.cloud = cloud.Cloud(None, self.paths, None, None, None)
+        self.cloud = cloud.Cloud(None, self.paths, None, distro, None)
         self.log = logging.getLogger("TestNoConfig")
         self.args = []
 
@@ -62,6 +64,11 @@ class TestConfig(TestCase):
         self.mock_remove = self.mocks.enter_context(
             mock.patch.object(cc_ca_certs, 'remove_default_ca_certs'))
 
+    def _fetch_distro(self, kind):
+        cls = distros.fetch(kind)
+        paths = helpers.Paths({})
+        return cls(kind, {}, paths)
+
     def test_no_trusted_list(self):
         """
         Test that no certificates are written if the 'trusted' key is not
@@ -275,7 +282,7 @@ class TestRemoveDefaultCaCerts(TestCase):
                 mock.patch.object(util, 'write_file'))
             mock_subp = mocks.enter_context(mock.patch.object(subp, 'subp'))
 
-            cc_ca_certs.remove_default_ca_certs()
+            cc_ca_certs.remove_default_ca_certs('ubuntu')
 
             mock_delete.assert_has_calls([
                 mock.call("/usr/share/ca-certificates/"),
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 92a33ec1..6b9c8377 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -239,6 +239,35 @@ class TestNtp(FilesystemMockingTestCase):
                     self.assertEqual(delta[distro][client][key],
                                      result[client][key])
 
+    def _get_expected_pools(self, pools, distro, client):
+        if client in ['ntp', 'chrony']:
+            if client == 'ntp' and distro == 'alpine':
+                # NTP for Alpine Linux is Busybox's ntp which does not
+                # support 'pool' lines in its configuration file.
+                expected_pools = []
+            else:
+                expected_pools = [
+                    'pool {0} iburst'.format(pool) for pool in pools]
+        elif client == 'systemd-timesyncd':
+            expected_pools = " ".join(pools)
+
+        return expected_pools
+
+    def _get_expected_servers(self, servers, distro, client):
+        if client in ['ntp', 'chrony']:
+            if client == 'ntp' and distro == 'alpine':
+                # NTP for Alpine Linux is Busybox's ntp which only supports
+                # 'server' lines without iburst option.
+                expected_servers = [
+                    'server {0}'.format(srv) for srv in servers]
+            else:
+                expected_servers = [
+                    'server {0} iburst'.format(srv) for srv in servers]
+        elif client == 'systemd-timesyncd':
+            expected_servers = " ".join(servers)
+
+        return expected_servers
+
     def test_ntp_handler_real_distro_ntp_templates(self):
         """Test ntp handler renders the shipped distro ntp client templates."""
         pools = ['0.mycompany.pool.ntp.org', '3.mycompany.pool.ntp.org']
@@ -269,27 +298,35 @@ class TestNtp(FilesystemMockingTestCase):
                 content = util.load_file(confpath)
                 if client in ['ntp', 'chrony']:
                     content_lines = content.splitlines()
-                    expected_servers = [
-                        'server {0} iburst'.format(srv) for srv in servers]
+                    expected_servers = self._get_expected_servers(servers,
+                                                                  distro,
+                                                                  client)
                     print('distro=%s client=%s' % (distro, client))
                     for sline in expected_servers:
                         self.assertIn(sline, content_lines,
                                       ('failed to render {0} conf'
                                        ' for distro:{1}'.format(client,
                                                                 distro)))
-                    expected_pools = [
-                        'pool {0} iburst'.format(pool) for pool in pools]
-                    for pline in expected_pools:
-                        self.assertIn(pline, content_lines,
-                                      ('failed to render {0} conf'
-                                       ' for distro:{1}'.format(client,
-                                                                distro)))
+                    expected_pools = self._get_expected_pools(pools, distro,
+                                                              client)
+                    if expected_pools != []:
+                        for pline in expected_pools:
+                            self.assertIn(pline, content_lines,
+                                          ('failed to render {0} conf'
+                                           ' for distro:{1}'.format(client,
+                                                                    distro)))
                 elif client == 'systemd-timesyncd':
+                    expected_servers = self._get_expected_servers(servers,
+                                                                  distro,
+                                                                  client)
+                    expected_pools = self._get_expected_pools(pools,
+                                                              distro,
+                                                              client)
                     expected_content = (
                         "# cloud-init generated file\n" +
                         "# See timesyncd.conf(5) for details.\n\n" +
-                        "[Time]\nNTP=%s %s \n" % (" ".join(servers),
-                                                  " ".join(pools)))
+                        "[Time]\nNTP=%s %s \n" % (expected_servers,
+                                                  expected_pools))
                     self.assertEqual(expected_content, content)
 
     def test_no_ntpcfg_does_nothing(self):
@@ -312,10 +349,20 @@ class TestNtp(FilesystemMockingTestCase):
                 confpath = ntpconfig['confpath']
                 m_select.return_value = ntpconfig
                 cc_ntp.handle('cc_ntp', valid_empty_config, mycloud, None, [])
-                pools = cc_ntp.generate_server_names(mycloud.distro.name)
-                self.assertEqual(
-                    "servers []\npools {0}\n".format(pools),
-                    util.load_file(confpath))
+                if distro == 'alpine':
+                    # _mock_ntp_client_config call above did not specify a
+                    # client value and so it defaults to "ntp" which on
+                    # Alpine Linux only supports servers and not pools.
+
+                    servers = cc_ntp.generate_server_names(mycloud.distro.name)
+                    self.assertEqual(
+                        "servers {0}\npools []\n".format(servers),
+                        util.load_file(confpath))
+                else:
+                    pools = cc_ntp.generate_server_names(mycloud.distro.name)
+                    self.assertEqual(
+                        "servers []\npools {0}\n".format(pools),
+                        util.load_file(confpath))
             self.assertNotIn('Invalid config:', self.logs.getvalue())
 
     @skipUnlessJsonSchema()
@@ -374,18 +421,19 @@ class TestNtp(FilesystemMockingTestCase):
         invalid_config = {
             'ntp': {'invalidkey': 1, 'pools': ['0.mycompany.pool.ntp.org']}}
         for distro in cc_ntp.distros:
-            mycloud = self._get_cloud(distro)
-            ntpconfig = self._mock_ntp_client_config(distro=distro)
-            confpath = ntpconfig['confpath']
-            m_select.return_value = ntpconfig
-            cc_ntp.handle('cc_ntp', invalid_config, mycloud, None, [])
-            self.assertIn(
-                "Invalid config:\nntp: Additional properties are not allowed "
-                "('invalidkey' was unexpected)",
-                self.logs.getvalue())
-            self.assertEqual(
-                "servers []\npools ['0.mycompany.pool.ntp.org']\n",
-                util.load_file(confpath))
+            if distro != 'alpine':
+                mycloud = self._get_cloud(distro)
+                ntpconfig = self._mock_ntp_client_config(distro=distro)
+                confpath = ntpconfig['confpath']
+                m_select.return_value = ntpconfig
+                cc_ntp.handle('cc_ntp', invalid_config, mycloud, None, [])
+                self.assertIn(
+                    "Invalid config:\nntp: Additional properties are not "
+                    "allowed ('invalidkey' was unexpected)",
+                    self.logs.getvalue())
+                self.assertEqual(
+                    "servers []\npools ['0.mycompany.pool.ntp.org']\n",
+                    util.load_file(confpath))
 
     @skipUnlessJsonSchema()
     @mock.patch('cloudinit.config.cc_ntp.select_ntp_client')
@@ -452,9 +500,22 @@ class TestNtp(FilesystemMockingTestCase):
             confpath = ntpconfig['confpath']
             service_name = ntpconfig['service_name']
             m_select.return_value = ntpconfig
-            pools = cc_ntp.generate_server_names(mycloud.distro.name)
-            # force uses systemd path
-            m_sysd.return_value = True
+
+            hosts = cc_ntp.generate_server_names(mycloud.distro.name)
+            uses_systemd = True
+            expected_service_call = ['systemctl', 'reload-or-restart',
+                                     service_name]
+            expected_content = "servers []\npools {0}\n".format(hosts)
+
+            if distro == 'alpine':
+                uses_systemd = False
+                expected_service_call = ['service', service_name, 'restart']
+                # _mock_ntp_client_config call above did not specify a client
+                # value and so it defaults to "ntp" which on Alpine Linux only
+                # supports servers and not pools.
+                expected_content = "servers {0}\npools []\n".format(hosts)
+
+            m_sysd.return_value = uses_systemd
             with mock.patch('cloudinit.config.cc_ntp.util') as m_util:
                 # allow use of util.mergemanydict
                 m_util.mergemanydict.side_effect = util.mergemanydict
@@ -465,11 +526,9 @@ class TestNtp(FilesystemMockingTestCase):
                     cfg['ntp']['enabled'])
                 cc_ntp.handle('notimportant', cfg, mycloud, None, None)
                 m_subp.subp.assert_called_with(
-                    ['systemctl', 'reload-or-restart',
-                     service_name], capture=True)
-            self.assertEqual(
-                "servers []\npools {0}\n".format(pools),
-                util.load_file(confpath))
+                    expected_service_call, capture=True)
+
+            self.assertEqual(expected_content, util.load_file(confpath))
 
     def test_opensuse_picks_chrony(self):
         """Test opensuse picks chrony or ntp on certain distro versions"""
diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py
index 0d8d17b9..93b24fdc 100644
--- a/tests/unittests/test_handler/test_handler_power_state.py
+++ b/tests/unittests/test_handler/test_handler_power_state.py
@@ -11,62 +11,63 @@ from cloudinit.tests.helpers import mock
 class TestLoadPowerState(t_help.TestCase):
     def test_no_config(self):
         # completely empty config should mean do nothing
-        (cmd, _timeout, _condition) = psc.load_power_state({})
+        (cmd, _timeout, _condition) = psc.load_power_state({}, 'ubuntu')
         self.assertIsNone(cmd)
 
     def test_irrelevant_config(self):
         # no power_state field in config should return None for cmd
-        (cmd, _timeout, _condition) = psc.load_power_state({'foo': 'bar'})
+        (cmd, _timeout, _condition) = psc.load_power_state({'foo': 'bar'},
+                                                           'ubuntu')
         self.assertIsNone(cmd)
 
     def test_invalid_mode(self):
         cfg = {'power_state': {'mode': 'gibberish'}}
-        self.assertRaises(TypeError, psc.load_power_state, cfg)
+        self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu')
 
         cfg = {'power_state': {'mode': ''}}
-        self.assertRaises(TypeError, psc.load_power_state, cfg)
+        self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu')
 
     def test_empty_mode(self):
         cfg = {'power_state': {'message': 'goodbye'}}
-        self.assertRaises(TypeError, psc.load_power_state, cfg)
+        self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu')
 
     def test_valid_modes(self):
         cfg = {'power_state': {}}
         for mode in ('halt', 'poweroff', 'reboot'):
             cfg['power_state']['mode'] = mode
-            check_lps_ret(psc.load_power_state(cfg), mode=mode)
+            check_lps_ret(psc.load_power_state(cfg, 'ubuntu'), mode=mode)
 
     def test_invalid_delay(self):
         cfg = {'power_state': {'mode': 'poweroff', 'delay': 'goodbye'}}
-        self.assertRaises(TypeError, psc.load_power_state, cfg)
+        self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu')
 
     def test_valid_delay(self):
         cfg = {'power_state': {'mode': 'poweroff', 'delay': ''}}
         for delay in ("now", "+1", "+30"):
             cfg['power_state']['delay'] = delay
-            check_lps_ret(psc.load_power_state(cfg))
+            check_lps_ret(psc.load_power_state(cfg, 'ubuntu'))
 
     def test_message_present(self):
         cfg = {'power_state': {'mode': 'poweroff', 'message': 'GOODBYE'}}
-        ret = psc.load_power_state(cfg)
-        check_lps_ret(psc.load_power_state(cfg))
+        ret = psc.load_power_state(cfg, 'ubuntu')
+        check_lps_ret(psc.load_power_state(cfg, 'ubuntu'))
         self.assertIn(cfg['power_state']['message'], ret[0])
 
     def test_no_message(self):
         # if message is not present, then no argument should be passed for it
         cfg = {'power_state': {'mode': 'poweroff'}}
-        (cmd, _timeout, _condition) = psc.load_power_state(cfg)
+        (cmd, _timeout, _condition) = psc.load_power_state(cfg, 'ubuntu')
         self.assertNotIn("", cmd)
-        check_lps_ret(psc.load_power_state(cfg))
+        check_lps_ret(psc.load_power_state(cfg, 'ubuntu'))
         self.assertTrue(len(cmd) == 3)
 
     def test_condition_null_raises(self):
         cfg = {'power_state': {'mode': 'poweroff', 'condition': None}}
-        self.assertRaises(TypeError, psc.load_power_state, cfg)
+        self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu')
 
     def test_condition_default_is_true(self):
         cfg = {'power_state': {'mode': 'poweroff'}}
-        _cmd, _timeout, cond = psc.load_power_state(cfg)
+        _cmd, _timeout, cond = psc.load_power_state(cfg, 'ubuntu')
         self.assertEqual(cond, True)
 
 
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index 99f0b06c..44292571 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -24,6 +24,7 @@ class GetSchemaTest(CiTestCase):
         schema = get_schema()
         self.assertCountEqual(
             [
+                'cc_apk_configure',
                 'cc_apt_configure',
                 'cc_bootcmd',
                 'cc_locale',
diff --git a/tools/render-cloudcfg b/tools/render-cloudcfg
index 9322b2c3..ed454840 100755
--- a/tools/render-cloudcfg
+++ b/tools/render-cloudcfg
@@ -4,8 +4,9 @@ import argparse
 import os
 import sys
 
-VARIANTS = ["amazon", "arch", "centos", "debian", "fedora", "freebsd",
-            "netbsd", "openbsd", "rhel", "suse", "ubuntu", "unknown"]
+VARIANTS = ["alpine", "amazon", "arch", "centos", "debian", "fedora",
+            "freebsd", "netbsd", "openbsd", "rhel", "suse", "ubuntu",
+            "unknown"]
 
 
 if "avoid-pep8-E402-import-not-top-of-file":
-- 
cgit v1.2.3


From 07104504ab5b30efd2d1f7a8c36effe18b8e5fe0 Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Tue, 25 Aug 2020 17:21:18 +0200
Subject: tox: bump the pylint version to 2.6.0 in the default run (#544)

Changes:

tox: bump the pylint version to 2.6.0 in the default run
Fix pylint 2.6.0 W0707 warnings (raise-missing-from)
---
 cloudinit/analyze/show.py                          |  2 +-
 cloudinit/cmd/devel/make_mime.py                   |  6 ++--
 cloudinit/config/cc_disk_setup.py                  | 36 ++++++++++++++--------
 cloudinit/config/cc_growpart.py                    |  8 ++---
 cloudinit/config/cc_power_state_change.py          | 12 +++++---
 cloudinit/config/cc_rsyslog.py                     |  6 ++--
 cloudinit/config/cc_set_hostname.py                |  2 +-
 cloudinit/config/cc_ubuntu_advantage.py            |  2 +-
 cloudinit/config/schema.py                         |  2 +-
 cloudinit/distros/__init__.py                      |  5 +--
 cloudinit/distros/arch.py                          |  4 +--
 cloudinit/distros/parsers/resolv_conf.py           |  7 +++--
 cloudinit/gpg.py                                   |  5 +--
 cloudinit/handlers/jinja_template.py               |  3 +-
 cloudinit/net/__init__.py                          |  7 +++--
 cloudinit/net/cmdline.py                           |  4 +--
 cloudinit/net/dhcp.py                              |  4 +--
 cloudinit/net/network_state.py                     | 19 +++++++-----
 cloudinit/sources/DataSourceAzure.py               |  2 +-
 cloudinit/sources/DataSourceEc2.py                 |  8 +++--
 cloudinit/sources/DataSourceIBMCloud.py            |  3 +-
 cloudinit/sources/DataSourceMAAS.py                |  3 +-
 cloudinit/sources/DataSourceOpenNebula.py          | 13 +++++---
 cloudinit/sources/DataSourceOpenStack.py           |  4 +--
 cloudinit/sources/DataSourceSmartOS.py             |  4 ++-
 cloudinit/sources/helpers/azure.py                 | 34 +++++++++++---------
 cloudinit/sources/helpers/netlink.py               |  2 +-
 cloudinit/sources/helpers/openstack.py             | 34 +++++++++++---------
 cloudinit/subp.py                                  |  3 +-
 cloudinit/url_helper.py                            |  6 ++--
 cloudinit/util.py                                  |  8 ++---
 tests/cloud_tests/platforms/azurecloud/instance.py |  7 +++--
 tests/cloud_tests/platforms/azurecloud/platform.py | 23 +++++++++-----
 tests/cloud_tests/platforms/ec2/instance.py        |  4 +--
 tests/cloud_tests/platforms/ec2/platform.py        | 18 ++++++-----
 tests/cloud_tests/platforms/lxd/instance.py        |  3 +-
 tests/cloud_tests/platforms/platforms.py           |  6 ++--
 tests/cloud_tests/testcases/__init__.py            |  6 ++--
 tools/mock-meta.py                                 | 14 +++++----
 tox.ini                                            |  2 +-
 40 files changed, 205 insertions(+), 136 deletions(-)

(limited to 'cloudinit/distros')

diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py
index 0c825b23..01a4d3e5 100644
--- a/cloudinit/analyze/show.py
+++ b/cloudinit/analyze/show.py
@@ -267,7 +267,7 @@ def gather_timestamps_using_systemd():
                 except OSError as err:
                     raise RuntimeError('Could not determine container boot '
                                        'time from /proc/1/cmdline. ({})'
-                                       .format(err))
+                                       .format(err)) from err
                 status = CONTAINER_CODE
             else:
                 status = FAIL_CODE
diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py
index 77e10540..4e6a5778 100755
--- a/cloudinit/cmd/devel/make_mime.py
+++ b/cloudinit/cmd/devel/make_mime.py
@@ -22,8 +22,10 @@ def file_content_type(text):
     try:
         filename, content_type = text.split(":", 1)
         return (open(filename, 'r'), filename, content_type.strip())
-    except ValueError:
-        raise argparse.ArgumentError(text, "Invalid value for %r" % (text))
+    except ValueError as e:
+        raise argparse.ArgumentError(
+            text, "Invalid value for %r" % (text)
+        ) from e
 
 
 def get_parser(parser=None):
diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py
index d957cfe3..a7bdc703 100644
--- a/cloudinit/config/cc_disk_setup.py
+++ b/cloudinit/config/cc_disk_setup.py
@@ -251,7 +251,9 @@ def enumerate_disk(device, nodeps=False):
     try:
         info, _err = subp.subp(lsblk_cmd)
     except Exception as e:
-        raise Exception("Failed during disk check for %s\n%s" % (device, e))
+        raise Exception(
+            "Failed during disk check for %s\n%s" % (device, e)
+        ) from e
 
     parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0]
 
@@ -313,7 +315,9 @@ def check_fs(device):
     try:
         out, _err = subp.subp(blkid_cmd, rcs=[0, 2])
     except Exception as e:
-        raise Exception("Failed during disk check for %s\n%s" % (device, e))
+        raise Exception(
+            "Failed during disk check for %s\n%s" % (device, e)
+        ) from e
 
     if out:
         if len(out.splitlines()) == 1:
@@ -428,8 +432,8 @@ def get_dyn_func(*args):
         else:
             return globals()[func_name]
 
-    except KeyError:
-        raise Exception("No such function %s to call!" % func_name)
+    except KeyError as e:
+        raise Exception("No such function %s to call!" % func_name) from e
 
 
 def get_hdd_size(device):
@@ -437,7 +441,7 @@ def get_hdd_size(device):
         size_in_bytes, _ = subp.subp([BLKDEV_CMD, '--getsize64', device])
         sector_size, _ = subp.subp([BLKDEV_CMD, '--getss', device])
     except Exception as e:
-        raise Exception("Failed to get %s size\n%s" % (device, e))
+        raise Exception("Failed to get %s size\n%s" % (device, e)) from e
 
     return int(size_in_bytes) / int(sector_size)
 
@@ -455,8 +459,9 @@ def check_partition_mbr_layout(device, layout):
     try:
         out, _err = subp.subp(prt_cmd, data="%s\n" % layout)
     except Exception as e:
-        raise Exception("Error running partition command on %s\n%s" % (
-                        device, e))
+        raise Exception(
+            "Error running partition command on %s\n%s" % (device, e)
+        ) from e
 
     found_layout = []
     for line in out.splitlines():
@@ -485,8 +490,9 @@ def check_partition_gpt_layout(device, layout):
     try:
         out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV)
     except Exception as e:
-        raise Exception("Error running partition command on %s\n%s" % (
-                        device, e))
+        raise Exception(
+            "Error running partition command on %s\n%s" % (device, e)
+        ) from e
 
     out_lines = iter(out.splitlines())
     # Skip header.  Output looks like:
@@ -657,8 +663,10 @@ def purge_disk(device):
             try:
                 LOG.info("Purging filesystem on /dev/%s", d['name'])
                 subp.subp(wipefs_cmd)
-            except Exception:
-                raise Exception("Failed FS purge of /dev/%s" % d['name'])
+            except Exception as e:
+                raise Exception(
+                    "Failed FS purge of /dev/%s" % d['name']
+                ) from e
 
     purge_disk_ptable(device)
 
@@ -700,7 +708,9 @@ def exec_mkpart_mbr(device, layout):
     try:
         subp.subp(prt_cmd, data="%s\n" % layout)
     except Exception as e:
-        raise Exception("Failed to partition device %s\n%s" % (device, e))
+        raise Exception(
+            "Failed to partition device %s\n%s" % (device, e)
+        ) from e
 
     read_parttbl(device)
 
@@ -997,6 +1007,6 @@ def mkfs(fs_cfg):
     try:
         subp.subp(fs_cmd, shell=shell)
     except Exception as e:
-        raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e))
+        raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) from e
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py
index c5d93f81..237c3d02 100644
--- a/cloudinit/config/cc_growpart.py
+++ b/cloudinit/config/cc_growpart.py
@@ -148,14 +148,14 @@ class ResizeGrowPart(object):
             if e.exit_code != 1:
                 util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)",
                             diskdev, partnum)
-                raise ResizeFailedException(e)
+                raise ResizeFailedException(e) from e
             return (before, before)
 
         try:
             subp.subp(["growpart", diskdev, partnum])
         except subp.ProcessExecutionError as e:
             util.logexc(LOG, "Failed: growpart %s %s", diskdev, partnum)
-            raise ResizeFailedException(e)
+            raise ResizeFailedException(e) from e
 
         return (before, get_size(partdev))
 
@@ -187,14 +187,14 @@ class ResizeGpart(object):
         except subp.ProcessExecutionError as e:
             if e.exit_code != 0:
                 util.logexc(LOG, "Failed: gpart recover %s", diskdev)
-                raise ResizeFailedException(e)
+                raise ResizeFailedException(e) from e
 
         before = get_size(partdev)
         try:
             subp.subp(["gpart", "resize", "-i", partnum, diskdev])
         except subp.ProcessExecutionError as e:
             util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev)
-            raise ResizeFailedException(e)
+            raise ResizeFailedException(e) from e
 
         # Since growing the FS requires a reboot, make sure we reboot
         # first when this module has finished.
diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py
index ab953a0d..6fcb8a7d 100644
--- a/cloudinit/config/cc_power_state_change.py
+++ b/cloudinit/config/cc_power_state_change.py
@@ -196,10 +196,11 @@ def load_power_state(cfg, distro_name):
 
     try:
         delay = convert_delay(delay, fmt=fmt, scale=scale)
-    except ValueError:
+    except ValueError as e:
         raise TypeError(
             "power_state[delay] must be 'now' or '+m' (minutes)."
-            " found '%s'." % delay)
+            " found '%s'." % delay
+        ) from e
 
     args = command + [delay]
     if message:
@@ -207,9 +208,10 @@ def load_power_state(cfg, distro_name):
 
     try:
         timeout = float(pstate.get('timeout', 30.0))
-    except ValueError:
-        raise ValueError("failed to convert timeout '%s' to float." %
-                         pstate['timeout'])
+    except ValueError as e:
+        raise ValueError(
+            "failed to convert timeout '%s' to float." % pstate['timeout']
+        ) from e
 
     condition = pstate.get("condition", True)
     if not isinstance(condition, (str, list, bool)):
diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py
index 1354885a..2a2bc931 100644
--- a/cloudinit/config/cc_rsyslog.py
+++ b/cloudinit/config/cc_rsyslog.py
@@ -347,8 +347,10 @@ class SyslogRemotesLine(object):
         if self.port:
             try:
                 int(self.port)
-            except ValueError:
-                raise ValueError("port '%s' is not an integer" % self.port)
+            except ValueError as e:
+                raise ValueError(
+                    "port '%s' is not an integer" % self.port
+                ) from e
 
         if not self.addr:
             raise ValueError("address is required")
diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py
index c13020d8..1d23d80d 100644
--- a/cloudinit/config/cc_set_hostname.py
+++ b/cloudinit/config/cc_set_hostname.py
@@ -85,7 +85,7 @@ def handle(name, cfg, cloud, log, _args):
     except Exception as e:
         msg = "Failed to set the hostname to %s (%s)" % (fqdn, hostname)
         util.logexc(log, msg)
-        raise SetHostnameError("%s: %s" % (msg, e))
+        raise SetHostnameError("%s: %s" % (msg, e)) from e
     write_json(prev_fn, {'hostname': hostname, 'fqdn': fqdn})
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py
index 35ded5db..d61dc655 100644
--- a/cloudinit/config/cc_ubuntu_advantage.py
+++ b/cloudinit/config/cc_ubuntu_advantage.py
@@ -115,7 +115,7 @@ def configure_ua(token=None, enable=None):
         msg = 'Failure attaching Ubuntu Advantage:\n{error}'.format(
             error=str(e))
         util.logexc(LOG, msg)
-        raise RuntimeError(msg)
+        raise RuntimeError(msg) from e
     enable_errors = []
     for service in enable:
         try:
diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py
index 2d8c7577..8a966aee 100644
--- a/cloudinit/config/schema.py
+++ b/cloudinit/config/schema.py
@@ -210,7 +210,7 @@ def validate_cloudconfig_file(config_path, schema, annotate=False):
         error = SchemaValidationError(errors)
         if annotate:
             print(annotated_cloudconfig_file({}, content, error.schema_errors))
-        raise error
+        raise error from e
     try:
         validate_cloudconfig_schema(
             cloudconfig, schema, strict=True)
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index effb4276..2537608f 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -607,10 +607,11 @@ class Distro(metaclass=abc.ABCMeta):
         lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name])
         try:
             cmd = next(tool for tool in lock_tools if subp.which(tool[0]))
-        except StopIteration:
+        except StopIteration as e:
             raise RuntimeError((
                 "Unable to lock user account '%s'. No tools available. "
-                "  Tried: %s.") % (name, [c[0] for c in lock_tools]))
+                "  Tried: %s.") % (name, [c[0] for c in lock_tools])
+            ) from e
         try:
             subp.subp(cmd)
         except Exception as e:
diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py
index 038aa9ac..967be168 100644
--- a/cloudinit/distros/arch.py
+++ b/cloudinit/distros/arch.py
@@ -61,9 +61,9 @@ class Distro(distros.Distro):
     def _write_network_config(self, netconfig):
         try:
             return self._supported_write_network_config(netconfig)
-        except RendererNotFoundError:
+        except RendererNotFoundError as e:
             # Fall back to old _write_network
-            raise NotImplementedError
+            raise NotImplementedError from e
 
     def _write_network(self, settings):
         entries = net_util.translate_network(settings)
diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py
index 299d54b5..62929d03 100644
--- a/cloudinit/distros/parsers/resolv_conf.py
+++ b/cloudinit/distros/parsers/resolv_conf.py
@@ -150,9 +150,10 @@ class ResolvConf(object):
                 tail = ''
             try:
                 (cfg_opt, cfg_values) = head.split(None, 1)
-            except (IndexError, ValueError):
-                raise IOError("Incorrectly formatted resolv.conf line %s"
-                              % (i + 1))
+            except (IndexError, ValueError) as e:
+                raise IOError(
+                    "Incorrectly formatted resolv.conf line %s" % (i + 1)
+                ) from e
             if cfg_opt not in ['nameserver', 'domain',
                                'search', 'sortlist', 'options']:
                 raise IOError("Unexpected resolv.conf option %s" % (cfg_opt))
diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py
index 72b5ac59..be0ca0ea 100644
--- a/cloudinit/gpg.py
+++ b/cloudinit/gpg.py
@@ -63,10 +63,11 @@ def recv_key(key, keyserver, retries=(1, 1)):
                 "Import failed with exit code %d, will try again in %ss",
                 error.exit_code, naplen)
             time.sleep(naplen)
-        except StopIteration:
+        except StopIteration as e:
             raise ValueError(
                 ("Failed to import key '%s' from keyserver '%s' "
-                 "after %d tries: %s") % (key, keyserver, trynum, error))
+                 "after %d tries: %s") % (key, keyserver, trynum, error)
+            ) from e
 
 
 def delete_key(key):
diff --git a/cloudinit/handlers/jinja_template.py b/cloudinit/handlers/jinja_template.py
index ce3accf6..aadfbf86 100644
--- a/cloudinit/handlers/jinja_template.py
+++ b/cloudinit/handlers/jinja_template.py
@@ -83,7 +83,8 @@ def render_jinja_payload_from_file(
         if e.errno == EACCES:
             raise RuntimeError(
                 'Cannot render jinja template vars. No read permission on'
-                " '%s'. Try sudo" % instance_data_file)
+                " '%s'. Try sudo" % instance_data_file
+            ) from e
 
     rendered_payload = render_jinja_payload(
         payload, payload_fn, instance_data, debug)
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 322af77b..e233149a 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -506,7 +506,9 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True):
     try:
         _rename_interfaces(extract_physdevs(netcfg))
     except RuntimeError as e:
-        raise RuntimeError('Failed to apply network config names: %s' % e)
+        raise RuntimeError(
+            'Failed to apply network config names: %s' % e
+        ) from e
 
 
 def interface_has_own_mac(ifname, strict=False):
@@ -965,7 +967,8 @@ class EphemeralIPv4Network(object):
             self.prefix = mask_to_net_prefix(prefix_or_mask)
         except ValueError as e:
             raise ValueError(
-                'Cannot setup network: {0}'.format(e))
+                'Cannot setup network: {0}'.format(e)
+            ) from e
 
         self.connectivity_url = connectivity_url
         self.interface = interface
diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py
index 7ca7262b..cc8dc17b 100755
--- a/cloudinit/net/cmdline.py
+++ b/cloudinit/net/cmdline.py
@@ -112,8 +112,8 @@ def _klibc_to_config_entry(content, mac_addrs=None):
     data = util.load_shell_content(content)
     try:
         name = data['DEVICE'] if 'DEVICE' in data else data['DEVICE6']
-    except KeyError:
-        raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data")
+    except KeyError as e:
+        raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data") from e
 
     # ipconfig on precise does not write PROTO
     # IPv6 config gives us IPV6PROTO, not PROTO.
diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py
index 9230cf7a..4394c68b 100644
--- a/cloudinit/net/dhcp.py
+++ b/cloudinit/net/dhcp.py
@@ -82,8 +82,8 @@ class EphemeralDHCPv4(object):
         try:
             leases = maybe_perform_dhcp_discovery(
                 self.iface, self.dhcp_log_func)
-        except InvalidDHCPLeaseFileError:
-            raise NoDHCPLeaseError()
+        except InvalidDHCPLeaseFileError as e:
+            raise NoDHCPLeaseError() from e
         if not leases:
             raise NoDHCPLeaseError()
         self.lease = leases[-1]
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index 7bfe8be0..b2f7d31e 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -297,9 +297,10 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta):
             command_type = command['type']
             try:
                 handler = self.command_handlers[command_type]
-            except KeyError:
-                raise RuntimeError("No handler found for"
-                                   " command '%s'" % command_type)
+            except KeyError as e:
+                raise RuntimeError(
+                    "No handler found for  command '%s'" % command_type
+                ) from e
             try:
                 handler(self, command)
             except InvalidCommand:
@@ -316,9 +317,10 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta):
                 continue
             try:
                 handler = self.command_handlers[command_type]
-            except KeyError:
-                raise RuntimeError("No handler found for"
-                                   " command '%s'" % command_type)
+            except KeyError as e:
+                raise RuntimeError(
+                    "No handler found for command '%s'" % command_type
+                ) from e
             try:
                 handler(self, command)
                 self._v2_common(command)
@@ -914,9 +916,10 @@ def _normalize_route(route):
     if metric:
         try:
             normal_route['metric'] = int(metric)
-        except ValueError:
+        except ValueError as e:
             raise TypeError(
-                'Route config metric {} is not an integer'.format(metric))
+                'Route config metric {} is not an integer'.format(metric)
+            ) from e
     return normal_route
 
 
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 86cc7c28..f3c6452b 100755
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -1147,7 +1147,7 @@ def read_azure_ovf(contents):
     except Exception as e:
         error_str = "Invalid ovf-env.xml: %s" % e
         report_diagnostic_event(error_str)
-        raise BrokenAzureDataSource(error_str)
+        raise BrokenAzureDataSource(error_str) from e
 
     results = find_child(dom.documentElement,
                          lambda n: n.localName == "ProvisioningSection")
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 355b4e2f..1d09c12a 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -617,9 +617,11 @@ def parse_strict_mode(cfgval):
     if sleep:
         try:
             sleep = int(sleep)
-        except ValueError:
-            raise ValueError("Invalid sleep '%s' in strict_id setting '%s': "
-                             "not an integer" % (sleep, cfgval))
+        except ValueError as e:
+            raise ValueError(
+                "Invalid sleep '%s' in strict_id setting '%s': not an integer"
+                % (sleep, cfgval)
+            ) from e
     else:
         sleep = None
 
diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py
index d2aa9a58..8d196185 100644
--- a/cloudinit/sources/DataSourceIBMCloud.py
+++ b/cloudinit/sources/DataSourceIBMCloud.py
@@ -303,7 +303,8 @@ def read_md():
     except sources.BrokenMetadata as e:
         raise RuntimeError(
             "Failed reading IBM config disk (platform=%s path=%s): %s" %
-            (platform, path, e))
+            (platform, path, e)
+        ) from e
 
     ret.update(results)
     return ret
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index c80f70c2..9156925f 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -226,7 +226,8 @@ def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None,
         except url_helper.UrlError as e:
             if e.code == 404 and not optional:
                 raise MAASSeedDirMalformed(
-                    "Missing required %s: %s" % (path, e))
+                    "Missing required %s: %s" % (path, e)
+                ) from e
             elif e.code != 404:
                 raise e
 
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 12b1f94f..45481938 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -399,18 +399,23 @@ def read_context_disk_dir(source_dir, distro, asuser=None):
         if asuser is not None:
             try:
                 pwd.getpwnam(asuser)
-            except KeyError:
+            except KeyError as e:
                 raise BrokenContextDiskDir(
                     "configured user '{user}' does not exist".format(
-                        user=asuser))
+                        user=asuser)
+                ) from e
         try:
             path = os.path.join(source_dir, 'context.sh')
             content = util.load_file(path)
             context = parse_shell_config(content, asuser=asuser)
         except subp.ProcessExecutionError as e:
-            raise BrokenContextDiskDir("Error processing context.sh: %s" % (e))
+            raise BrokenContextDiskDir(
+                "Error processing context.sh: %s" % (e)
+            ) from e
         except IOError as e:
-            raise NonContextDiskDir("Error reading context.sh: %s" % (e))
+            raise NonContextDiskDir(
+                "Error reading context.sh: %s" % (e)
+            ) from e
     else:
         raise NonContextDiskDir("Missing context.sh")
 
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index bf539091..d4b43f44 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -194,10 +194,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
                         'timeout': url_params.timeout_seconds})
         except openstack.NonReadable as e:
             raise sources.InvalidMetaDataException(str(e))
-        except (openstack.BrokenMetadata, IOError):
+        except (openstack.BrokenMetadata, IOError) as e:
             msg = 'Broken metadata address {addr}'.format(
                 addr=self.metadata_address)
-            raise sources.InvalidMetaDataException(msg)
+            raise sources.InvalidMetaDataException(msg) from e
         return result
 
 
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 843b3a2a..f1f903bc 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -413,7 +413,9 @@ class JoyentMetadataClient(object):
                 response.append(byte)
             except OSError as exc:
                 if exc.errno == errno.EAGAIN:
-                    raise JoyentMetadataTimeoutException(msg % as_ascii())
+                    raise JoyentMetadataTimeoutException(
+                        msg % as_ascii()
+                    ) from exc
                 raise
 
     def _write(self, msg):
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 6156c75b..b968a96f 100755
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -98,8 +98,10 @@ def get_boot_telemetry():
     LOG.debug("Collecting boot telemetry")
     try:
         kernel_start = float(time.time()) - float(util.uptime())
-    except ValueError:
-        raise RuntimeError("Failed to determine kernel start timestamp")
+    except ValueError as e:
+        raise RuntimeError(
+            "Failed to determine kernel start timestamp"
+        ) from e
 
     try:
         out, _ = subp.subp(['/bin/systemctl',
@@ -116,12 +118,13 @@ def get_boot_telemetry():
 
         user_start = kernel_start + (float(tsm) / 1000000)
     except subp.ProcessExecutionError as e:
-        raise RuntimeError("Failed to get UserspaceTimestampMonotonic: %s"
-                           % e)
+        raise RuntimeError(
+            "Failed to get UserspaceTimestampMonotonic: %s" % e
+        ) from e
     except ValueError as e:
-        raise RuntimeError("Failed to parse "
-                           "UserspaceTimestampMonotonic from systemd: %s"
-                           % e)
+        raise RuntimeError(
+            "Failed to parse UserspaceTimestampMonotonic from systemd: %s" % e
+        ) from e
 
     try:
         out, _ = subp.subp(['/bin/systemctl', 'show',
@@ -137,12 +140,14 @@ def get_boot_telemetry():
 
         cloudinit_activation = kernel_start + (float(tsm) / 1000000)
     except subp.ProcessExecutionError as e:
-        raise RuntimeError("Failed to get InactiveExitTimestampMonotonic: %s"
-                           % e)
+        raise RuntimeError(
+            "Failed to get InactiveExitTimestampMonotonic: %s" % e
+        ) from e
     except ValueError as e:
-        raise RuntimeError("Failed to parse "
-                           "InactiveExitTimestampMonotonic from systemd: %s"
-                           % e)
+        raise RuntimeError(
+            "Failed to parse InactiveExitTimestampMonotonic from systemd: %s"
+            % e
+        ) from e
 
     evt = events.ReportingEvent(
         BOOT_EVENT_TYPE, 'boot-telemetry',
@@ -642,9 +647,10 @@ class WALinuxAgentShim:
             try:
                 name = os.path.basename(hook_file).replace('.json', '')
                 dhcp_options[name] = json.loads(util.load_file((hook_file)))
-            except ValueError:
+            except ValueError as e:
                 raise ValueError(
-                    '{_file} is not valid JSON data'.format(_file=hook_file))
+                    '{_file} is not valid JSON data'.format(_file=hook_file)
+                ) from e
         return dhcp_options
 
     @staticmethod
diff --git a/cloudinit/sources/helpers/netlink.py b/cloudinit/sources/helpers/netlink.py
index a74a3588..c2ad587b 100644
--- a/cloudinit/sources/helpers/netlink.py
+++ b/cloudinit/sources/helpers/netlink.py
@@ -74,7 +74,7 @@ def create_bound_netlink_socket():
         netlink_socket.setblocking(0)
     except socket.error as e:
         msg = "Exception during netlink socket create: %s" % e
-        raise NetlinkCreateSocketError(msg)
+        raise NetlinkCreateSocketError(msg) from e
     LOG.debug("Created netlink socket")
     return netlink_socket
 
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 1050efb0..65e020c5 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -280,8 +280,9 @@ class BaseReader(metaclass=abc.ABCMeta):
                 try:
                     data = translator(data)
                 except Exception as e:
-                    raise BrokenMetadata("Failed to process "
-                                         "path %s: %s" % (path, e))
+                    raise BrokenMetadata(
+                        "Failed to process path %s: %s" % (path, e)
+                    ) from e
             if found:
                 results[name] = data
 
@@ -291,8 +292,9 @@ class BaseReader(metaclass=abc.ABCMeta):
             try:
                 metadata['random_seed'] = base64.b64decode(random_seed)
             except (ValueError, TypeError) as e:
-                raise BrokenMetadata("Badly formatted metadata"
-                                     " random_seed entry: %s" % e)
+                raise BrokenMetadata(
+                    "Badly formatted metadata random_seed entry: %s" % e
+                ) from e
 
         # load any files that were provided
         files = {}
@@ -304,8 +306,9 @@ class BaseReader(metaclass=abc.ABCMeta):
             try:
                 files[path] = self._read_content_path(item)
             except Exception as e:
-                raise BrokenMetadata("Failed to read provided "
-                                     "file %s: %s" % (path, e))
+                raise BrokenMetadata(
+                    "Failed to read provided file %s: %s" % (path, e)
+                ) from e
         results['files'] = files
 
         # The 'network_config' item in metadata is a content pointer
@@ -317,8 +320,9 @@ class BaseReader(metaclass=abc.ABCMeta):
                 content = self._read_content_path(net_item, decode=True)
                 results['network_config'] = content
             except IOError as e:
-                raise BrokenMetadata("Failed to read network"
-                                     " configuration: %s" % (e))
+                raise BrokenMetadata(
+                    "Failed to read network configuration: %s" % (e)
+                ) from e
 
         # To openstack, user can specify meta ('nova boot --meta=key=value')
         # and those will appear under metadata['meta'].
@@ -370,8 +374,9 @@ class ConfigDriveReader(BaseReader):
             try:
                 return util.load_json(self._path_read(path))
             except Exception as e:
-                raise BrokenMetadata("Failed to process "
-                                     "path %s: %s" % (path, e))
+                raise BrokenMetadata(
+                    "Failed to process path %s: %s" % (path, e)
+                ) from e
 
     def read_v1(self):
         """Reads a version 1 formatted location.
@@ -395,16 +400,17 @@ class ConfigDriveReader(BaseReader):
                 path = found[name]
                 try:
                     contents = self._path_read(path)
-                except IOError:
-                    raise BrokenMetadata("Failed to read: %s" % path)
+                except IOError as e:
+                    raise BrokenMetadata("Failed to read: %s" % path) from e
                 try:
                     # Disable not-callable pylint check; pylint isn't able to
                     # determine that every member of FILES_V1 has a callable in
                     # the appropriate position
                     md[key] = translator(contents)  # pylint: disable=E1102
                 except Exception as e:
-                    raise BrokenMetadata("Failed to process "
-                                         "path %s: %s" % (path, e))
+                    raise BrokenMetadata(
+                        "Failed to process path %s: %s" % (path, e)
+                    ) from e
             else:
                 md[key] = copy.deepcopy(default)
 
diff --git a/cloudinit/subp.py b/cloudinit/subp.py
index 804ef3ca..3e4efa42 100644
--- a/cloudinit/subp.py
+++ b/cloudinit/subp.py
@@ -262,7 +262,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True,
         raise ProcessExecutionError(
             cmd=args, reason=e, errno=e.errno,
             stdout="-" if decode else b"-",
-            stderr="-" if decode else b"-")
+            stderr="-" if decode else b"-"
+        ) from e
     finally:
         if devnull_fp:
             devnull_fp.close()
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index f3c0cf9c..caa88435 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -95,7 +95,7 @@ def read_file_or_url(url, **kwargs):
             code = e.errno
             if e.errno == ENOENT:
                 code = NOT_FOUND
-            raise UrlError(cause=e, code=code, headers=None, url=url)
+            raise UrlError(cause=e, code=code, headers=None, url=url) from e
         return FileResponse(file_path, contents=contents)
     else:
         return readurl(url, **kwargs)
@@ -575,8 +575,8 @@ def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret,
                   timestamp=None):
     try:
         import oauthlib.oauth1 as oauth1
-    except ImportError:
-        raise NotImplementedError('oauth support is not available')
+    except ImportError as e:
+        raise NotImplementedError('oauth support is not available') from e
 
     if timestamp:
         timestamp = str(timestamp)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index dd263803..cf9e349f 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -359,7 +359,7 @@ def decomp_gzip(data, quiet=True, decode=True):
         if quiet:
             return data
         else:
-            raise DecompressionError(str(e))
+            raise DecompressionError(str(e)) from e
 
 
 def extract_usergroup(ug_pair):
@@ -1363,7 +1363,7 @@ def chownbyname(fname, user=None, group=None):
         if group:
             gid = grp.getgrnam(group).gr_gid
     except KeyError as e:
-        raise OSError("Unknown user or group: %s" % (e))
+        raise OSError("Unknown user or group: %s" % (e)) from e
     chownbyid(fname, uid, gid)
 
 
@@ -2387,8 +2387,8 @@ def human2bytes(size):
 
     try:
         num = float(num)
-    except ValueError:
-        raise ValueError("'%s' is not valid input." % size_in)
+    except ValueError as e:
+        raise ValueError("'%s' is not valid input." % size_in) from e
 
     if num < 0:
         raise ValueError("'%s': cannot be negative" % size_in)
diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py
index a136cf0d..eedbaae8 100644
--- a/tests/cloud_tests/platforms/azurecloud/instance.py
+++ b/tests/cloud_tests/platforms/azurecloud/instance.py
@@ -134,9 +134,10 @@ class AzureCloudInstance(Instance):
                                  self.vm_name, vm_params)
             LOG.debug('creating instance %s from image_id=%s', self.vm_name,
                       self.image_id)
-        except CloudError:
-            raise RuntimeError('failed creating instance:\n{}'.format(
-                traceback.format_exc()))
+        except CloudError as e:
+            raise RuntimeError(
+                'failed creating instance:\n{}'.format(traceback.format_exc())
+            ) from e
 
         if wait:
             self.instance.wait()
diff --git a/tests/cloud_tests/platforms/azurecloud/platform.py b/tests/cloud_tests/platforms/azurecloud/platform.py
index cb62a74b..a664f612 100644
--- a/tests/cloud_tests/platforms/azurecloud/platform.py
+++ b/tests/cloud_tests/platforms/azurecloud/platform.py
@@ -59,9 +59,12 @@ class AzureCloudPlatform(Platform):
             self.vnet = self._create_vnet()
             self.subnet = self._create_subnet()
             self.nic = self._create_nic()
-        except CloudError:
-            raise RuntimeError('failed creating a resource:\n{}'.format(
-                traceback.format_exc()))
+        except CloudError as e:
+            raise RuntimeError(
+                'failed creating a resource:\n{}'.format(
+                    traceback.format_exc()
+                )
+            ) from e
 
     def create_instance(self, properties, config, features,
                         image_id, user_data=None):
@@ -105,8 +108,10 @@ class AzureCloudPlatform(Platform):
             if image_id.find('__') > 0:
                 image_id = image_id.split('__')[1]
                 LOG.debug('image_id shortened to %s', image_id)
-        except KeyError:
-            raise RuntimeError('no images found for %s' % img_conf['release'])
+        except KeyError as e:
+            raise RuntimeError(
+                'no images found for %s' % img_conf['release']
+            ) from e
 
         return AzureCloudImage(self, img_conf, image_id)
 
@@ -140,9 +145,11 @@ class AzureCloudPlatform(Platform):
                 secret=azure_creds['clientSecret'],
                 tenant=azure_creds['tenantId'])
             return credentials, subscription_id
-        except KeyError:
-            raise RuntimeError('Please configure Azure service principal'
-                               ' credentials in %s' % cred_file)
+        except KeyError as e:
+            raise RuntimeError(
+                'Please configure Azure service principal'
+                ' credentials in %s' % cred_file
+            ) from e
 
     def _create_resource_group(self):
         """Create resource group"""
diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py
index ab6037b1..d2e84047 100644
--- a/tests/cloud_tests/platforms/ec2/instance.py
+++ b/tests/cloud_tests/platforms/ec2/instance.py
@@ -49,11 +49,11 @@ class EC2Instance(Instance):
             # OutputBytes comes from platform._decode_console_output_as_bytes
             response = self.instance.console_output()
             return response['OutputBytes']
-        except KeyError:
+        except KeyError as e:
             if 'Output' in response:
                 msg = ("'OutputBytes' did not exist in console_output() but "
                        "'Output' did: %s..." % response['Output'][0:128])
-                raise util.PlatformError('console_log', msg)
+                raise util.PlatformError('console_log', msg) from e
             return ('No Console Output [%s]' % self.instance).encode()
 
     def destroy(self):
diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py
index 7a3d0fe0..b61a2ffb 100644
--- a/tests/cloud_tests/platforms/ec2/platform.py
+++ b/tests/cloud_tests/platforms/ec2/platform.py
@@ -35,12 +35,14 @@ class EC2Platform(Platform):
             self.ec2_resource = b3session.resource('ec2')
             self.ec2_region = b3session.region_name
             self.key_name = self._upload_public_key(config)
-        except botocore.exceptions.NoRegionError:
+        except botocore.exceptions.NoRegionError as e:
             raise RuntimeError(
-                'Please configure default region in $HOME/.aws/config')
-        except botocore.exceptions.NoCredentialsError:
+                'Please configure default region in $HOME/.aws/config'
+            ) from e
+        except botocore.exceptions.NoCredentialsError as e:
             raise RuntimeError(
-                'Please configure ec2 credentials in $HOME/.aws/credentials')
+                'Please configure ec2 credentials in $HOME/.aws/credentials'
+            ) from e
 
         self.vpc = self._create_vpc()
         self.internet_gateway = self._create_internet_gateway()
@@ -125,8 +127,10 @@ class EC2Platform(Platform):
 
         try:
             image_ami = image['id']
-        except KeyError:
-            raise RuntimeError('No images found for %s!' % img_conf['release'])
+        except KeyError as e:
+            raise RuntimeError(
+                'No images found for %s!' % img_conf['release']
+            ) from e
 
         LOG.debug('found image: %s', image_ami)
         image = EC2Image(self, img_conf, image_ami)
@@ -195,7 +199,7 @@ class EC2Platform(Platform):
                 CidrBlock=self.ipv4_cidr,
                 AmazonProvidedIpv6CidrBlock=True)
         except botocore.exceptions.ClientError as e:
-            raise RuntimeError(e)
+            raise RuntimeError(e) from e
 
         vpc.wait_until_available()
         self._tag_resource(vpc)
diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py
index b27b9848..2b973a08 100644
--- a/tests/cloud_tests/platforms/lxd/instance.py
+++ b/tests/cloud_tests/platforms/lxd/instance.py
@@ -175,7 +175,8 @@ class LXDInstance(Instance):
             raise PlatformError(
                 "console log",
                 "Console log failed [%d]: stdout=%s stderr=%s" % (
-                    e.exit_code, e.stdout, e.stderr))
+                    e.exit_code, e.stdout, e.stderr)
+            ) from e
 
     def reboot(self, wait=True):
         """Reboot instance."""
diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py
index 58f65e52..ac3b6563 100644
--- a/tests/cloud_tests/platforms/platforms.py
+++ b/tests/cloud_tests/platforms/platforms.py
@@ -74,8 +74,10 @@ class Platform(object):
 
         try:
             return tmirror.json_entries[0]
-        except IndexError:
-            raise RuntimeError('no images found with filter: %s' % img_filter)
+        except IndexError as e:
+            raise RuntimeError(
+                'no images found with filter: %s' % img_filter
+            ) from e
 
 
 class FilterMirror(mirrors.BasicMirrorWriter):
diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py
index e8c371ca..bb9785d3 100644
--- a/tests/cloud_tests/testcases/__init__.py
+++ b/tests/cloud_tests/testcases/__init__.py
@@ -21,8 +21,10 @@ def discover_test(test_name):
         config.name_sanitize(test_name))
     try:
         testmod = importlib.import_module(testmod_name)
-    except NameError:
-        raise ValueError('no test verifier found at: {}'.format(testmod_name))
+    except NameError as e:
+        raise ValueError(
+            'no test verifier found at: {}'.format(testmod_name)
+        ) from e
 
     found = [mod for name, mod in inspect.getmembers(testmod)
              if (inspect.isclass(mod)
diff --git a/tools/mock-meta.py b/tools/mock-meta.py
index a58e0260..9dd067b9 100755
--- a/tools/mock-meta.py
+++ b/tools/mock-meta.py
@@ -258,12 +258,14 @@ class MetaDataHandler(object):
                 try:
                     key_id = int(mybe_key)
                     key_name = key_ids[key_id]
-                except ValueError:
-                    raise WebException(hclient.BAD_REQUEST,
-                                       "%s: not an integer" % mybe_key)
-                except IndexError:
-                    raise WebException(hclient.NOT_FOUND,
-                                       "Unknown key id %r" % mybe_key)
+                except ValueError as e:
+                    raise WebException(
+                        hclient.BAD_REQUEST, "%s: not an integer" % mybe_key
+                    ) from e
+                except IndexError as e:
+                    raise WebException(
+                        hclient.NOT_FOUND, "Unknown key id %r" % mybe_key
+                    ) from e
                 # Extract the possible sub-params
                 result = traverse(nparams[1:], {
                     "openssh-key": "\n".join(avail_keys[key_name]),
diff --git a/tox.ini b/tox.ini
index f619dbf5..a92c63e0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,7 +23,7 @@ setenv =
 basepython = python3
 deps =
     # requirements
-    pylint==2.5.3
+    pylint==2.6.0
     # test-requirements because unit tests are now present in cloudinit tree
     -r{toxinidir}/test-requirements.txt
     -r{toxinidir}/integration-requirements.txt
-- 
cgit v1.2.3