diff options
Diffstat (limited to 'cloudinit')
96 files changed, 1122 insertions, 820 deletions
diff --git a/cloudinit/analyze/dump.py b/cloudinit/analyze/dump.py index 939c3126..62ad51fe 100644 --- a/cloudinit/analyze/dump.py +++ b/cloudinit/analyze/dump.py @@ -4,6 +4,7 @@ import calendar from datetime import datetime import sys +from cloudinit import subp from cloudinit import util stage_to_description = { @@ -51,7 +52,7 @@ def parse_timestamp(timestampstr): def parse_timestamp_from_date(timestampstr): - out, _ = util.subp(['date', '+%s.%3N', '-d', timestampstr]) + out, _ = subp.subp(['date', '+%s.%3N', '-d', timestampstr]) timestamp = out.strip() return float(timestamp) diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py index fb152b1d..cca1fa7f 100644 --- a/cloudinit/analyze/show.py +++ b/cloudinit/analyze/show.py @@ -11,6 +11,7 @@ import os import time import sys +from cloudinit import subp from cloudinit import util from cloudinit.distros import uses_systemd @@ -155,7 +156,7 @@ class SystemctlReader(object): :return: whether the subp call failed or not ''' try: - value, err = util.subp(self.args, capture=True) + value, err = subp.subp(self.args, capture=True) if err: return err self.epoch = value @@ -215,7 +216,7 @@ def gather_timestamps_using_dmesg(): with gather_timestamps_using_systemd ''' try: - data, _ = util.subp(['dmesg'], capture=True) + data, _ = subp.subp(['dmesg'], capture=True) split_entries = data[0].splitlines() for i in split_entries: if i.decode('UTF-8').find('user') != -1: diff --git a/cloudinit/analyze/tests/test_boot.py b/cloudinit/analyze/tests/test_boot.py index f4001c14..f69423c3 100644 --- a/cloudinit/analyze/tests/test_boot.py +++ b/cloudinit/analyze/tests/test_boot.py @@ -25,7 +25,7 @@ class TestDistroChecker(CiTestCase): m_get_linux_distro, m_is_FreeBSD): self.assertEqual(err_code, dist_check_timestamp()) - @mock.patch('cloudinit.util.subp', return_value=(0, 1)) + @mock.patch('cloudinit.subp.subp', return_value=(0, 1)) def test_subp_fails(self, m_subp): self.assertEqual(err_code, dist_check_timestamp()) @@ -42,7 +42,7 @@ class TestSystemCtlReader(CiTestCase): with self.assertRaises(RuntimeError): reader.parse_epoch_as_float() - @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None)) def test_systemctl_works_correctly_threshold(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') self.assertEqual(1.0, reader.parse_epoch_as_float()) @@ -50,12 +50,12 @@ class TestSystemCtlReader(CiTestCase): self.assertTrue(thresh < 1e-6) self.assertTrue(thresh > (-1 * 1e-6)) - @mock.patch('cloudinit.util.subp', return_value=('U=0', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=0', None)) def test_systemctl_succeed_zero(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') self.assertEqual(0.0, reader.parse_epoch_as_float()) - @mock.patch('cloudinit.util.subp', return_value=('U=1', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1', None)) def test_systemctl_succeed_distinct(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') val1 = reader.parse_epoch_as_float() @@ -64,13 +64,13 @@ class TestSystemCtlReader(CiTestCase): val2 = reader2.parse_epoch_as_float() self.assertNotEqual(val1, val2) - @mock.patch('cloudinit.util.subp', return_value=('100', None)) + @mock.patch('cloudinit.subp.subp', return_value=('100', None)) def test_systemctl_epoch_not_splittable(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') with self.assertRaises(IndexError): reader.parse_epoch_as_float() - @mock.patch('cloudinit.util.subp', return_value=('U=foobar', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=foobar', None)) def test_systemctl_cannot_convert_epoch_to_float(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') with self.assertRaises(ValueError): @@ -130,7 +130,7 @@ class TestAnalyzeBoot(CiTestCase): self.assertEqual(err_string, data) @mock.patch("cloudinit.util.is_container", return_value=True) - @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None)) def test_container_no_ci_log_line(self, m_is_container, m_subp): path = os.path.dirname(os.path.abspath(__file__)) log_path = path + '/boot-test.log' @@ -148,7 +148,7 @@ class TestAnalyzeBoot(CiTestCase): self.assertEqual(FAIL_CODE, finish_code) @mock.patch("cloudinit.util.is_container", return_value=True) - @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None)) @mock.patch('cloudinit.analyze.__main__._get_events', return_value=[{ 'name': 'init-local', 'description': 'starting search', 'timestamp': 100000}]) diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py index d6fbd381..dac1efb6 100644 --- a/cloudinit/analyze/tests/test_dump.py +++ b/cloudinit/analyze/tests/test_dump.py @@ -5,7 +5,8 @@ from textwrap import dedent from cloudinit.analyze.dump import ( dump_events, parse_ci_logline, parse_timestamp) -from cloudinit.util import which, write_file +from cloudinit.util import write_file +from cloudinit.subp import which from cloudinit.tests.helpers import CiTestCase, mock, skipIf diff --git a/cloudinit/cmd/clean.py b/cloudinit/cmd/clean.py index 30e49de0..928a8eea 100644 --- a/cloudinit/cmd/clean.py +++ b/cloudinit/cmd/clean.py @@ -10,9 +10,8 @@ import os import sys from cloudinit.stages import Init -from cloudinit.util import ( - ProcessExecutionError, del_dir, del_file, get_config_logfiles, - is_link, subp) +from cloudinit.subp import (ProcessExecutionError, subp) +from cloudinit.util import (del_dir, del_file, get_config_logfiles, is_link) def error(msg): diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py index 4c086b51..51c61cca 100644 --- a/cloudinit/cmd/devel/logs.py +++ b/cloudinit/cmd/devel/logs.py @@ -12,8 +12,8 @@ import sys from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE from cloudinit.temp_utils import tempdir -from cloudinit.util import ( - ProcessExecutionError, chdir, copy, ensure_dir, subp, write_file) +from cloudinit.subp import (ProcessExecutionError, subp) +from cloudinit.util import (chdir, copy, ensure_dir, write_file) CLOUDINIT_LOGS = ['/var/log/cloud-init.log', '/var/log/cloud-init-output.log'] diff --git a/cloudinit/cmd/devel/tests/test_logs.py b/cloudinit/cmd/devel/tests/test_logs.py index d2dfa8de..ddfd58e1 100644 --- a/cloudinit/cmd/devel/tests/test_logs.py +++ b/cloudinit/cmd/devel/tests/test_logs.py @@ -8,7 +8,8 @@ from cloudinit.cmd.devel import logs from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE from cloudinit.tests.helpers import ( FilesystemMockingTestCase, mock, wrap_and_call) -from cloudinit.util import ensure_dir, load_file, subp, write_file +from cloudinit.subp import subp +from cloudinit.util import ensure_dir, load_file, write_file @mock.patch('cloudinit.cmd.devel.logs.os.getuid') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index b1c7b471..73d8719f 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -17,6 +17,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit import gpg from cloudinit import log as logging +from cloudinit import subp from cloudinit import templater from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -431,7 +432,7 @@ def _should_configure_on_empty_apt(): # if no config was provided, should apt configuration be done? if util.system_is_snappy(): return False, "system is snappy." - if not (util.which('apt-get') or util.which('apt')): + if not (subp.which('apt-get') or subp.which('apt')): return False, "no apt commands." return True, "Apt is available." @@ -478,7 +479,7 @@ def apply_apt(cfg, cloud, target): def debconf_set_selections(selections, target=None): if not selections.endswith(b'\n'): selections += b'\n' - util.subp(['debconf-set-selections'], data=selections, target=target, + subp.subp(['debconf-set-selections'], data=selections, target=target, capture=True) @@ -503,7 +504,7 @@ def dpkg_reconfigure(packages, target=None): "but cannot be unconfigured: %s", unhandled) if len(to_config): - util.subp(['dpkg-reconfigure', '--frontend=noninteractive'] + + subp.subp(['dpkg-reconfigure', '--frontend=noninteractive'] + list(to_config), data=None, target=target, capture=True) @@ -546,7 +547,7 @@ def apply_debconf_selections(cfg, target=None): def clean_cloud_init(target): """clean out any local cloud-init config""" flist = glob.glob( - util.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) + subp.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) LOG.debug("cleaning cloud-init config from: %s", flist) for dpkg_cfg in flist: @@ -575,7 +576,7 @@ def rename_apt_lists(new_mirrors, target, arch): """rename_apt_lists - rename apt lists to preserve old cache data""" default_mirrors = get_default_mirrors(arch) - pre = util.target_path(target, APT_LISTS) + pre = subp.target_path(target, APT_LISTS) for (name, omirror) in default_mirrors.items(): nmirror = new_mirrors.get(name) if not nmirror: @@ -694,8 +695,8 @@ def add_apt_key_raw(key, target=None): """ LOG.debug("Adding key:\n'%s'", key) try: - util.subp(['apt-key', 'add', '-'], data=key.encode(), target=target) - except util.ProcessExecutionError: + subp.subp(['apt-key', 'add', '-'], data=key.encode(), target=target) + except subp.ProcessExecutionError: LOG.exception("failed to add apt GPG Key to apt keyring") raise @@ -758,13 +759,13 @@ def add_apt_sources(srcdict, cloud, target=None, template_params=None, if aa_repo_match(source): try: - util.subp(["add-apt-repository", source], target=target) - except util.ProcessExecutionError: + subp.subp(["add-apt-repository", source], target=target) + except subp.ProcessExecutionError: LOG.exception("add-apt-repository failed.") raise continue - sourcefn = util.target_path(target, ent['filename']) + sourcefn = subp.target_path(target, ent['filename']) try: contents = "%s\n" % (source) util.write_file(sourcefn, contents, omode="a") diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 6813f534..246e4497 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -16,6 +16,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_ALWAYS from cloudinit import temp_utils +from cloudinit import subp from cloudinit import util frequency = PER_ALWAYS @@ -99,7 +100,7 @@ def handle(name, cfg, cloud, log, _args): if iid: env['INSTANCE_ID'] = str(iid) cmd = ['/bin/sh', tmpf.name] - util.subp(cmd, env=env, capture=False) + subp.subp(cmd, env=env, capture=False) except Exception: util.logexc(log, "Failed to run bootcmd module %s", name) raise diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 0b4352c8..9fdaeba1 100755 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -39,6 +39,7 @@ Valid configuration options for this module are: """ from cloudinit.distros import ug_util +from cloudinit import subp from cloudinit import util distros = ['ubuntu', 'debian'] @@ -93,6 +94,6 @@ def handle(name, cfg, cloud, log, args): if len(shcmd): cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] log.debug("Setting byobu to %s", value) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 64bc900e..7617a8ea 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -36,6 +36,7 @@ can be removed from the system with the configuration option import os +from cloudinit import subp from cloudinit import util CA_CERT_PATH = "/usr/share/ca-certificates/" @@ -51,7 +52,7 @@ def update_ca_certs(): """ Updates the CA certificate cache on the current machine. """ - util.subp(["update-ca-certificates"], capture=False) + subp.subp(["update-ca-certificates"], capture=False) def add_ca_certs(certs): @@ -85,7 +86,7 @@ def remove_default_ca_certs(): util.delete_dir_contents(CA_CERT_SYSTEM_PATH) util.write_file(CA_CERT_CONFIG, "", mode=0o644) debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" - util.subp(('debconf-set-selections', '-'), debconf_sel) + subp.subp(('debconf-set-selections', '-'), debconf_sel) def handle(name, cfg, _cloud, log, _args): diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 03285ef0..e1f51fce 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -76,7 +76,9 @@ import itertools import json import os +from cloudinit import subp from cloudinit import templater +from cloudinit import temp_utils from cloudinit import url_helper from cloudinit import util @@ -282,7 +284,32 @@ def run_chef(chef_cfg, log): cmd.extend(CHEF_EXEC_DEF_ARGS) else: cmd.extend(CHEF_EXEC_DEF_ARGS) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) + + +def subp_blob_in_tempfile(blob, *args, **kwargs): + """Write blob to a tempfile, and call subp with args, kwargs. Then cleanup. + + 'basename' as a kwarg allows providing the basename for the file. + The 'args' argument to subp will be updated with the full path to the + filename as the first argument. + """ + basename = kwargs.pop('basename', "subp_blob") + + if len(args) == 0 and 'args' not in kwargs: + args = [tuple()] + + # Use tmpdir over tmpfile to avoid 'text file busy' on execute + with temp_utils.tempdir(needs_exe=True) as tmpd: + tmpf = os.path.join(tmpd, basename) + if 'args' in kwargs: + kwargs['args'] = [tmpf] + list(kwargs['args']) + else: + args = list(args) + args[0] = [tmpf] + args[0] + + util.write_file(tmpf, blob, mode=0o700) + return subp.subp(*args, **kwargs) def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None): @@ -305,7 +332,7 @@ def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None): else: args = ['-v', omnibus_version] content = url_helper.readurl(url=url, retries=retries).contents - return util.subp_blob_in_tempfile( + return subp_blob_in_tempfile( blob=content, args=args, basename='chef-omnibus-install', capture=False) @@ -354,11 +381,11 @@ def install_chef_from_gems(ruby_version, chef_version, distro): if not os.path.exists('/usr/bin/ruby'): util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') if chef_version: - util.subp(['/usr/bin/gem', 'install', 'chef', + subp.subp(['/usr/bin/gem', 'install', 'chef', '-v %s' % chef_version, '--no-ri', '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) else: - util.subp(['/usr/bin/gem', 'install', 'chef', + subp.subp(['/usr/bin/gem', 'install', 'chef', '--no-ri', '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) diff --git a/cloudinit/config/cc_disable_ec2_metadata.py b/cloudinit/config/cc_disable_ec2_metadata.py index 885b3138..dff93245 100644 --- a/cloudinit/config/cc_disable_ec2_metadata.py +++ b/cloudinit/config/cc_disable_ec2_metadata.py @@ -26,6 +26,7 @@ by default. disable_ec2_metadata: <true/false> """ +from cloudinit import subp from cloudinit import util from cloudinit.settings import PER_ALWAYS @@ -40,15 +41,15 @@ def handle(name, cfg, _cloud, log, _args): disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False) if disabled: reject_cmd = None - if util.which('ip'): + if subp.which('ip'): reject_cmd = REJECT_CMD_IP - elif util.which('ifconfig'): + elif subp.which('ifconfig'): reject_cmd = REJECT_CMD_IF else: log.error(('Neither "route" nor "ip" command found, unable to ' 'manipulate routing table')) return - util.subp(reject_cmd, capture=False) + subp.subp(reject_cmd, capture=False) else: log.debug(("Skipping module named %s," " disabling the ec2 route not enabled"), name) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 45925755..d957cfe3 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -99,6 +99,7 @@ specified using ``filesystem``. from cloudinit.settings import PER_INSTANCE from cloudinit import util +from cloudinit import subp import logging import os import shlex @@ -106,13 +107,13 @@ import shlex frequency = PER_INSTANCE # Define the commands to use -UDEVADM_CMD = util.which('udevadm') -SFDISK_CMD = util.which("sfdisk") -SGDISK_CMD = util.which("sgdisk") -LSBLK_CMD = util.which("lsblk") -BLKID_CMD = util.which("blkid") -BLKDEV_CMD = util.which("blockdev") -WIPEFS_CMD = util.which("wipefs") +UDEVADM_CMD = subp.which('udevadm') +SFDISK_CMD = subp.which("sfdisk") +SGDISK_CMD = subp.which("sgdisk") +LSBLK_CMD = subp.which("lsblk") +BLKID_CMD = subp.which("blkid") +BLKDEV_CMD = subp.which("blockdev") +WIPEFS_CMD = subp.which("wipefs") LANG_C_ENV = {'LANG': 'C'} @@ -248,7 +249,7 @@ def enumerate_disk(device, nodeps=False): info = None try: - info, _err = util.subp(lsblk_cmd) + info, _err = subp.subp(lsblk_cmd) except Exception as e: raise Exception("Failed during disk check for %s\n%s" % (device, e)) @@ -310,7 +311,7 @@ def check_fs(device): blkid_cmd = [BLKID_CMD, '-c', '/dev/null', device] try: - out, _err = util.subp(blkid_cmd, rcs=[0, 2]) + out, _err = subp.subp(blkid_cmd, rcs=[0, 2]) except Exception as e: raise Exception("Failed during disk check for %s\n%s" % (device, e)) @@ -433,8 +434,8 @@ def get_dyn_func(*args): def get_hdd_size(device): try: - size_in_bytes, _ = util.subp([BLKDEV_CMD, '--getsize64', device]) - sector_size, _ = util.subp([BLKDEV_CMD, '--getss', device]) + size_in_bytes, _ = subp.subp([BLKDEV_CMD, '--getsize64', device]) + sector_size, _ = subp.subp([BLKDEV_CMD, '--getss', device]) except Exception as e: raise Exception("Failed to get %s size\n%s" % (device, e)) @@ -452,7 +453,7 @@ def check_partition_mbr_layout(device, layout): read_parttbl(device) prt_cmd = [SFDISK_CMD, "-l", device] try: - out, _err = util.subp(prt_cmd, data="%s\n" % layout) + out, _err = subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: raise Exception("Error running partition command on %s\n%s" % ( device, e)) @@ -482,7 +483,7 @@ def check_partition_mbr_layout(device, layout): def check_partition_gpt_layout(device, layout): prt_cmd = [SGDISK_CMD, '-p', device] try: - out, _err = util.subp(prt_cmd, update_env=LANG_C_ENV) + out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV) except Exception as e: raise Exception("Error running partition command on %s\n%s" % ( device, e)) @@ -655,7 +656,7 @@ def purge_disk(device): wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']] try: LOG.info("Purging filesystem on /dev/%s", d['name']) - util.subp(wipefs_cmd) + subp.subp(wipefs_cmd) except Exception: raise Exception("Failed FS purge of /dev/%s" % d['name']) @@ -682,7 +683,7 @@ def read_parttbl(device): blkdev_cmd = [BLKDEV_CMD, '--rereadpt', device] util.udevadm_settle() try: - util.subp(blkdev_cmd) + subp.subp(blkdev_cmd) except Exception as e: util.logexc(LOG, "Failed reading the partition table %s" % e) @@ -697,7 +698,7 @@ def exec_mkpart_mbr(device, layout): # Create the partitions prt_cmd = [SFDISK_CMD, "--Linux", "--unit=S", "--force", device] try: - util.subp(prt_cmd, data="%s\n" % layout) + subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: raise Exception("Failed to partition device %s\n%s" % (device, e)) @@ -706,16 +707,16 @@ def exec_mkpart_mbr(device, layout): def exec_mkpart_gpt(device, layout): try: - util.subp([SGDISK_CMD, '-Z', device]) + subp.subp([SGDISK_CMD, '-Z', device]) for index, (partition_type, (start, end)) in enumerate(layout): index += 1 - util.subp([SGDISK_CMD, + subp.subp([SGDISK_CMD, '-n', '{}:{}:{}'.format(index, start, end), device]) if partition_type is not None: # convert to a 4 char (or more) string right padded with 0 # 82 -> 8200. 'Linux' -> 'Linux' pinput = str(partition_type).ljust(4, "0") - util.subp( + subp.subp( [SGDISK_CMD, '-t', '{}:{}'.format(index, pinput), device]) except Exception: LOG.warning("Failed to partition device %s", device) @@ -967,9 +968,9 @@ def mkfs(fs_cfg): fs_cmd) else: # Find the mkfs command - mkfs_cmd = util.which("mkfs.%s" % fs_type) + mkfs_cmd = subp.which("mkfs.%s" % fs_type) if not mkfs_cmd: - mkfs_cmd = util.which("mk%s" % fs_type) + mkfs_cmd = subp.which("mk%s" % fs_type) if not mkfs_cmd: LOG.warning("Cannot create fstype '%s'. No mkfs.%s command", @@ -994,7 +995,7 @@ def mkfs(fs_cfg): LOG.debug("Creating file system %s on %s", label, device) LOG.debug(" Using cmd: %s", str(fs_cmd)) try: - util.subp(fs_cmd, shell=shell) + subp.subp(fs_cmd, shell=shell) except Exception as e: raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index b342e04d..b1d99f97 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -25,7 +25,7 @@ import os from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS -from cloudinit import util +from cloudinit import subp frequency = PER_ALWAYS @@ -43,9 +43,9 @@ def is_upstart_system(): del myenv['UPSTART_SESSION'] check_cmd = ['initctl', 'version'] try: - (out, _err) = util.subp(check_cmd, env=myenv) + (out, _err) = subp.subp(check_cmd, env=myenv) return 'upstart' in out - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: LOG.debug("'%s' returned '%s', not using upstart", ' '.join(check_cmd), e.exit_code) return False @@ -66,7 +66,7 @@ def handle(name, _cfg, cloud, log, args): for n in event_names: cmd = ['initctl', 'emit', str(n), 'CLOUD_CFG=%s' % cfgpath] try: - util.subp(cmd) + subp.subp(cmd) except Exception as e: # TODO(harlowja), use log exception from utils?? log.warning("Emission of upstart event %s failed due to: %s", n, e) diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py index 0a135bbe..77984bca 100644 --- a/cloudinit/config/cc_fan.py +++ b/cloudinit/config/cc_fan.py @@ -39,6 +39,7 @@ If cloud-init sees a ``fan`` entry in cloud-config it will: from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -62,8 +63,8 @@ def stop_update_start(service, config_file, content, systemd=False): def run(cmd, msg): try: - return util.subp(cmd, capture=True) - except util.ProcessExecutionError as e: + return subp.subp(cmd, capture=True) + except subp.ProcessExecutionError as e: LOG.warning("failed: %s (%s): %s", service, cmd, e) return False @@ -94,7 +95,7 @@ def handle(name, cfg, cloud, log, args): util.write_file(mycfg.get('config_path'), mycfg.get('config'), omode="w") distro = cloud.distro - if not util.which('fanctl'): + if not subp.which('fanctl'): distro.install_packages(['ubuntu-fan']) stop_update_start( diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 1b512a06..c5d93f81 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -70,6 +70,7 @@ import stat from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS +from cloudinit import subp from cloudinit import util frequency = PER_ALWAYS @@ -131,19 +132,19 @@ class ResizeGrowPart(object): myenv['LANG'] = 'C' try: - (out, _err) = util.subp(["growpart", "--help"], env=myenv) + (out, _err) = subp.subp(["growpart", "--help"], env=myenv) if re.search(r"--update\s+", out): return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass return False def resize(self, diskdev, partnum, partdev): before = get_size(partdev) try: - util.subp(["growpart", '--dry-run', diskdev, partnum]) - except util.ProcessExecutionError as e: + subp.subp(["growpart", '--dry-run', diskdev, partnum]) + except subp.ProcessExecutionError as e: if e.exit_code != 1: util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)", diskdev, partnum) @@ -151,8 +152,8 @@ class ResizeGrowPart(object): return (before, before) try: - util.subp(["growpart", diskdev, partnum]) - except util.ProcessExecutionError as e: + subp.subp(["growpart", diskdev, partnum]) + except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: growpart %s %s", diskdev, partnum) raise ResizeFailedException(e) @@ -165,11 +166,11 @@ class ResizeGpart(object): myenv['LANG'] = 'C' try: - (_out, err) = util.subp(["gpart", "help"], env=myenv, rcs=[0, 1]) + (_out, err) = subp.subp(["gpart", "help"], env=myenv, rcs=[0, 1]) if re.search(r"gpart recover ", err): return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass return False @@ -182,16 +183,16 @@ class ResizeGpart(object): be recovered. """ try: - util.subp(["gpart", "recover", diskdev]) - except util.ProcessExecutionError as e: + subp.subp(["gpart", "recover", diskdev]) + except subp.ProcessExecutionError as e: if e.exit_code != 0: util.logexc(LOG, "Failed: gpart recover %s", diskdev) raise ResizeFailedException(e) before = get_size(partdev) try: - util.subp(["gpart", "resize", "-i", partnum, diskdev]) - except util.ProcessExecutionError as e: + subp.subp(["gpart", "resize", "-i", partnum, diskdev]) + except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev) raise ResizeFailedException(e) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 7888464e..eb03c664 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -43,8 +43,9 @@ seeded with empty values, and install_devices_empty is set to true. import os +from cloudinit import subp from cloudinit import util -from cloudinit.util import ProcessExecutionError +from cloudinit.subp import ProcessExecutionError distros = ['ubuntu', 'debian'] @@ -59,7 +60,7 @@ def fetch_idevs(log): try: # get the root disk where the /boot directory resides. - disk = util.subp(['grub-probe', '-t', 'disk', '/boot'], + disk = subp.subp(['grub-probe', '-t', 'disk', '/boot'], capture=True)[0].strip() except ProcessExecutionError as e: # grub-common may not be installed, especially on containers @@ -84,7 +85,7 @@ def fetch_idevs(log): try: # check if disk exists and use udevadm to fetch symlinks - devices = util.subp( + devices = subp.subp( ['udevadm', 'info', '--root', '--query=symlink', disk], capture=True )[0].strip().split() @@ -135,7 +136,7 @@ def handle(name, cfg, _cloud, log, _args): (idevs, idevs_empty)) try: - util.subp(['debconf-set-selections'], dconf_sel) + subp.subp(['debconf-set-selections'], dconf_sel) except Exception: util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg") diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index 3d2ded3d..0f2be52b 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -33,6 +33,7 @@ key can be used. By default ``ssh-dss`` keys are not written to console. import os from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util frequency = PER_INSTANCE @@ -64,7 +65,7 @@ def handle(name, cfg, cloud, log, _args): try: cmd = [helper_path, ','.join(fp_blacklist), ','.join(key_blacklist)] - (stdout, _stderr) = util.subp(cmd) + (stdout, _stderr) = subp.subp(cmd) util.multi_log("%s\n" % (stdout.strip()), stderr=False, console=True) except Exception: diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index a9c04d86..299c4d01 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -61,6 +61,7 @@ from io import BytesIO from configobj import ConfigObj from cloudinit import type_utils +from cloudinit import subp from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -116,7 +117,7 @@ def handle(_name, cfg, cloud, log, _args): log.debug("Wrote landscape config file to %s", LSC_CLIENT_CFG_FILE) util.write_file(LS_DEFAULT_FILE, "RUN=1\n") - util.subp(["service", "landscape-client", "restart"]) + subp.subp(["service", "landscape-client", "restart"]) def merge_together(objs): diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 151a9844..7129c9c6 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -48,6 +48,7 @@ lxd-bridge will be configured accordingly. """ from cloudinit import log as logging +from cloudinit import subp from cloudinit import util import os @@ -85,16 +86,16 @@ def handle(name, cfg, cloud, log, args): # Install the needed packages packages = [] - if not util.which("lxd"): + if not subp.which("lxd"): packages.append('lxd') - if init_cfg.get("storage_backend") == "zfs" and not util.which('zfs'): + if init_cfg.get("storage_backend") == "zfs" and not subp.which('zfs'): packages.append('zfsutils-linux') if len(packages): try: cloud.distro.install_packages(packages) - except util.ProcessExecutionError as exc: + except subp.ProcessExecutionError as exc: log.warning("failed to install packages %s: %s", packages, exc) return @@ -104,20 +105,20 @@ def handle(name, cfg, cloud, log, args): 'network_address', 'network_port', 'storage_backend', 'storage_create_device', 'storage_create_loop', 'storage_pool', 'trust_password') - util.subp(['lxd', 'waitready', '--timeout=300']) + subp.subp(['lxd', 'waitready', '--timeout=300']) cmd = ['lxd', 'init', '--auto'] for k in init_keys: if init_cfg.get(k): cmd.extend(["--%s=%s" % (k.replace('_', '-'), str(init_cfg[k]))]) - util.subp(cmd) + subp.subp(cmd) # Set up lxd-bridge if bridge config is given dconf_comm = "debconf-communicate" if bridge_cfg: net_name = bridge_cfg.get("name", _DEFAULT_NETWORK_NAME) if os.path.exists("/etc/default/lxd-bridge") \ - and util.which(dconf_comm): + and subp.which(dconf_comm): # Bridge configured through packaging debconf = bridge_to_debconf(bridge_cfg) @@ -127,7 +128,7 @@ def handle(name, cfg, cloud, log, args): log.debug("Setting lxd debconf via " + dconf_comm) data = "\n".join(["set %s %s" % (k, v) for k, v in debconf.items()]) + "\n" - util.subp(['debconf-communicate'], data) + subp.subp(['debconf-communicate'], data) except Exception: util.logexc(log, "Failed to run '%s' for lxd with" % dconf_comm) @@ -137,7 +138,7 @@ def handle(name, cfg, cloud, log, args): # Run reconfigure log.debug("Running dpkg-reconfigure for lxd") - util.subp(['dpkg-reconfigure', 'lxd', + subp.subp(['dpkg-reconfigure', 'lxd', '--frontend=noninteractive']) else: # Built-in LXD bridge support @@ -264,7 +265,7 @@ def _lxc(cmd): env = {'LC_ALL': 'C', 'HOME': os.environ.get('HOME', '/root'), 'USER': os.environ.get('USER', 'root')} - util.subp(['lxc'] + list(cmd) + ["--force-local"], update_env=env) + subp.subp(['lxc'] + list(cmd) + ["--force-local"], update_env=env) def maybe_cleanup_default(net_name, did_init, create, attach, @@ -286,7 +287,7 @@ def maybe_cleanup_default(net_name, did_init, create, attach, try: _lxc(["network", "delete", net_name]) LOG.debug(msg, net_name, succeeded) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code != 1: raise e LOG.debug(msg, net_name, fail_assume_enoent) @@ -296,7 +297,7 @@ def maybe_cleanup_default(net_name, did_init, create, attach, try: _lxc(["profile", "device", "remove", profile, nic_name]) LOG.debug(msg, nic_name, profile, succeeded) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code != 1: raise e LOG.debug(msg, nic_name, profile, fail_assume_enoent) diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py index 351183f1..41ea4fc9 100644 --- a/cloudinit/config/cc_mcollective.py +++ b/cloudinit/config/cc_mcollective.py @@ -56,6 +56,7 @@ import io from configobj import ConfigObj from cloudinit import log as logging +from cloudinit import subp from cloudinit import util PUBCERT_FILE = "/etc/mcollective/ssl/server-public.pem" @@ -140,6 +141,6 @@ def handle(name, cfg, cloud, log, _args): configure(config=mcollective_cfg['conf']) # restart mcollective to handle updated config - util.subp(['service', 'mcollective', 'restart'], capture=False) + subp.subp(['service', 'mcollective', 'restart'], capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 85a89cd1..e57d1b1f 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -69,6 +69,7 @@ import os.path import re from cloudinit import type_utils +from cloudinit import subp from cloudinit import util # Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0 @@ -252,8 +253,8 @@ def create_swapfile(fname: str, size: str) -> None: 'count=%s' % size] try: - util.subp(cmd, capture=True) - except util.ProcessExecutionError as e: + subp.subp(cmd, capture=True) + except subp.ProcessExecutionError as e: LOG.warning(errmsg, fname, size, method, e) util.del_file(fname) @@ -267,15 +268,15 @@ def create_swapfile(fname: str, size: str) -> None: else: try: create_swap(fname, size, "fallocate") - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: LOG.warning(errmsg, fname, size, "dd", e) LOG.warning("Will attempt with dd.") create_swap(fname, size, "dd") util.chmod(fname, 0o600) try: - util.subp(['mkswap', fname]) - except util.ProcessExecutionError: + subp.subp(['mkswap', fname]) + except subp.ProcessExecutionError: util.del_file(fname) raise @@ -538,9 +539,9 @@ def handle(_name, cfg, cloud, log, _args): for cmd in activate_cmds: fmt = "Activate mounts: %s:" + ' '.join(cmd) try: - util.subp(cmd) + subp.subp(cmd) log.debug(fmt, "PASS") - except util.ProcessExecutionError: + except subp.ProcessExecutionError: log.warning(fmt, "FAIL") util.logexc(log, fmt, "FAIL") diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 3b2c2020..7d3f73ff 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -14,6 +14,7 @@ from cloudinit import log as logging from cloudinit import temp_utils from cloudinit import templater from cloudinit import type_utils +from cloudinit import subp from cloudinit import util from cloudinit.config.schema import get_schema_doc, validate_cloudconfig_schema from cloudinit.settings import PER_INSTANCE @@ -307,7 +308,7 @@ def select_ntp_client(ntp_client, distro): if distro_ntp_client == "auto": for client in distro.preferred_ntp_clients: cfg = distro_cfg.get(client) - if util.which(cfg.get('check_exe')): + if subp.which(cfg.get('check_exe')): LOG.debug('Selected NTP client "%s", already installed', client) clientcfg = cfg @@ -336,7 +337,7 @@ def install_ntp_client(install_func, packages=None, check_exe="ntpd"): @param check_exe: string. The name of a binary that indicates the package the specified package is already installed. """ - if util.which(check_exe): + if subp.which(check_exe): return if packages is None: packages = ['ntp'] @@ -431,7 +432,7 @@ def reload_ntp(service, systemd=False): cmd = ['systemctl', 'reload-or-restart', service] else: cmd = ['service', service, 'restart'] - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def supplemental_schema_validation(ntp_config): @@ -543,7 +544,7 @@ def handle(name, cfg, cloud, log, _args): try: reload_ntp(ntp_client_config['service_name'], systemd=cloud.distro.uses_systemd()) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: LOG.exception("Failed to reload/start ntp service: %s", e) raise diff --git a/cloudinit/config/cc_package_update_upgrade_install.py b/cloudinit/config/cc_package_update_upgrade_install.py index 86afffef..036baf85 100644 --- a/cloudinit/config/cc_package_update_upgrade_install.py +++ b/cloudinit/config/cc_package_update_upgrade_install.py @@ -43,6 +43,7 @@ import os import time from cloudinit import log as logging +from cloudinit import subp from cloudinit import util REBOOT_FILE = "/var/run/reboot-required" @@ -57,7 +58,7 @@ def _multi_cfg_bool_get(cfg, *keys): def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): - util.subp(REBOOT_CMD) + subp.subp(REBOOT_CMD) start = time.time() wait_time = initial_sleep for _i in range(0, wait_attempts): diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 3e81a3c7..41ffb46c 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -56,6 +56,7 @@ import subprocess import time from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util frequency = PER_INSTANCE @@ -71,7 +72,7 @@ def givecmdline(pid): # PID COMM ARGS # 1 init /bin/init -- if util.is_FreeBSD(): - (output, _err) = util.subp(['procstat', '-c', str(pid)]) + (output, _err) = subp.subp(['procstat', '-c', str(pid)]) line = output.splitlines()[1] m = re.search(r'\d+ (\w|\.|-)+\s+(/\w.+)', line) return m.group(2) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index c01f5b8f..635c73bc 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -83,6 +83,7 @@ import yaml from io import StringIO from cloudinit import helpers +from cloudinit import subp from cloudinit import util PUPPET_CONF_PATH = '/etc/puppet/puppet.conf' @@ -105,14 +106,14 @@ class PuppetConstants(object): def _autostart_puppet(log): # Set puppet to automatically start if os.path.exists('/etc/default/puppet'): - util.subp(['sed', '-i', + subp.subp(['sed', '-i', '-e', 's/^START=.*/START=yes/', '/etc/default/puppet'], capture=False) elif os.path.exists('/bin/systemctl'): - util.subp(['/bin/systemctl', 'enable', 'puppet.service'], + subp.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False) elif os.path.exists('/sbin/chkconfig'): - util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) + subp.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) else: log.warning(("Sorry we do not know how to enable" " puppet services on this system")) @@ -203,6 +204,6 @@ def handle(name, cfg, cloud, log, _args): _autostart_puppet(log) # Start puppetd - util.subp(['service', 'puppet', 'start'], capture=False) + subp.subp(['service', 'puppet', 'start'], capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 01dfc125..8de4db30 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -19,6 +19,7 @@ from textwrap import dedent from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_ALWAYS +from cloudinit import subp from cloudinit import util NOBLOCK = "noblock" @@ -88,11 +89,11 @@ def _resize_zfs(mount_point, devpth): def _get_dumpfs_output(mount_point): - return util.subp(['dumpfs', '-m', mount_point])[0] + return subp.subp(['dumpfs', '-m', mount_point])[0] def _get_gpart_output(part): - return util.subp(['gpart', 'show', part])[0] + return subp.subp(['gpart', 'show', part])[0] def _can_skip_resize_ufs(mount_point, devpth): @@ -306,8 +307,8 @@ def handle(name, cfg, _cloud, log, args): def do_resize(resize_cmd, log): try: - util.subp(resize_cmd) - except util.ProcessExecutionError: + subp.subp(resize_cmd) + except subp.ProcessExecutionError: util.logexc(log, "Failed to resize filesystem (cmd=%s)", resize_cmd) raise # TODO(harlowja): Should we add a fsck check after this to make diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 28c79b83..28d62e9d 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -39,6 +39,7 @@ Subscription`` example config. """ from cloudinit import log as logging +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -173,7 +174,7 @@ class SubscriptionManager(object): try: _sub_man_cli(cmd) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: return False return True @@ -200,7 +201,7 @@ class SubscriptionManager(object): try: return_out = _sub_man_cli(cmd, logstring_val=True)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.stdout == "": self.log_warn("Registration failed due " "to: {0}".format(e.stderr)) @@ -223,7 +224,7 @@ class SubscriptionManager(object): # Attempting to register the system only try: return_out = _sub_man_cli(cmd, logstring_val=True)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.stdout == "": self.log_warn("Registration failed due " "to: {0}".format(e.stderr)) @@ -246,7 +247,7 @@ class SubscriptionManager(object): try: return_out = _sub_man_cli(cmd)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.stdout.rstrip() != '': for line in e.stdout.split("\n"): if line != '': @@ -264,7 +265,7 @@ class SubscriptionManager(object): cmd = ['attach', '--auto'] try: return_out = _sub_man_cli(cmd)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: self.log_warn("Auto-attach failed with: {0}".format(e)) return False for line in return_out.split("\n"): @@ -341,7 +342,7 @@ class SubscriptionManager(object): "system: %s", (", ".join(pool_list)) .replace('--pool=', '')) return True - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: self.log_warn("Unable to attach pool {0} " "due to {1}".format(pool, e)) return False @@ -414,7 +415,7 @@ class SubscriptionManager(object): try: _sub_man_cli(cmd) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: self.log_warn("Unable to alter repos due to {0}".format(e)) return False @@ -432,11 +433,11 @@ class SubscriptionManager(object): def _sub_man_cli(cmd, logstring_val=False): ''' - Uses the prefered cloud-init subprocess def of util.subp + Uses the prefered cloud-init subprocess def of subp.subp and runs subscription-manager. Breaking this to a separate function for later use in mocking and unittests ''' - return util.subp(['subscription-manager'] + cmd, + return subp.subp(['subscription-manager'] + cmd, logstring=logstring_val) diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 5df0137d..1354885a 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -182,6 +182,7 @@ import os import re from cloudinit import log as logging +from cloudinit import subp from cloudinit import util DEF_FILENAME = "20-cloud-config.conf" @@ -215,7 +216,7 @@ def reload_syslog(command=DEF_RELOAD, systemd=False): cmd = ['service', service, 'restart'] else: cmd = command - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def load_config(cfg): @@ -429,7 +430,7 @@ def handle(name, cfg, cloud, log, _args): restarted = reload_syslog( command=mycfg[KEYNAME_RELOAD], systemd=cloud.distro.uses_systemd()), - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: restarted = False log.warning("Failed to reload syslog", e) diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 5dd8de37..b61876aa 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -45,7 +45,7 @@ specify them with ``pkg_name``, ``service_name`` and ``config_dir``. import os -from cloudinit import safeyaml, util +from cloudinit import safeyaml, subp, util from cloudinit.distros import rhel_util @@ -130,6 +130,6 @@ def handle(name, cfg, cloud, log, _args): # restart salt-minion. 'service' will start even if not started. if it # was started, it needs to be restarted for config change. - util.subp(['service', const.srv_name, 'restart'], capture=False) + subp.subp(['service', const.srv_name, 'restart'], capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index b65f3ed9..4fb9b44e 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -65,6 +65,7 @@ from io import BytesIO from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util frequency = PER_INSTANCE @@ -92,14 +93,14 @@ def handle_random_seed_command(command, required, env=None): return cmd = command[0] - if not util.which(cmd): + if not subp.which(cmd): if required: raise ValueError( "command '{cmd}' not found but required=true".format(cmd=cmd)) else: LOG.debug("command '%s' not found for seed_command", cmd) return - util.subp(command, env=env, capture=False) + subp.subp(command, env=env, capture=False) def handle(name, cfg, cloud, log, _args): diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 7b7aa885..d6b5682d 100755 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -83,6 +83,7 @@ import sys from cloudinit.distros import ug_util from cloudinit import log as logging from cloudinit.ssh_util import update_ssh_config +from cloudinit import subp from cloudinit import util from string import ascii_letters, digits @@ -128,7 +129,7 @@ def handle_ssh_pwauth(pw_auth, service_cmd=None, service_name="ssh"): cmd = list(service_cmd) + ["restart", service_name] else: cmd = list(service_cmd) + [service_name, "restart"] - util.subp(cmd) + subp.subp(cmd) LOG.debug("Restarted the SSH daemon.") @@ -247,6 +248,6 @@ def chpasswd(distro, plist_in, hashed=False): distro.set_passwd(u, p, hashed=hashed) else: cmd = ['chpasswd'] + (['-e'] if hashed else []) - util.subp(cmd, plist_in) + subp.subp(cmd, plist_in) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py index 8178562e..20ed7d2f 100644 --- a/cloudinit/config/cc_snap.py +++ b/cloudinit/config/cc_snap.py @@ -12,6 +12,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_INSTANCE from cloudinit.subp import prepend_base_command +from cloudinit import subp from cloudinit import util @@ -175,7 +176,7 @@ def add_assertions(assertions): LOG.debug('Snap acking: %s', asrt.split('\n')[0:2]) util.write_file(ASSERTIONS_FILE, combined.encode('utf-8')) - util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) + subp.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) def run_commands(commands): @@ -204,8 +205,8 @@ def run_commands(commands): for command in fixed_snap_commands: shell = isinstance(command, str) try: - util.subp(command, shell=shell, status_cb=sys.stderr.write) - except util.ProcessExecutionError as e: + subp.subp(command, shell=shell, status_cb=sys.stderr.write) + except subp.ProcessExecutionError as e: cmd_failures.append(str(e)) if cmd_failures: msg = 'Failures running snap commands:\n{cmd_failures}'.format( diff --git a/cloudinit/config/cc_spacewalk.py b/cloudinit/config/cc_spacewalk.py index 1020e944..95083607 100644 --- a/cloudinit/config/cc_spacewalk.py +++ b/cloudinit/config/cc_spacewalk.py @@ -27,7 +27,7 @@ For more information about spacewalk see: https://fedorahosted.org/spacewalk/ activation_key: <key> """ -from cloudinit import util +from cloudinit import subp distros = ['redhat', 'fedora'] @@ -41,9 +41,9 @@ def is_registered(): # assume we aren't registered; which is sorta ghetto... already_registered = False try: - util.subp(['rhn-profile-sync', '--verbose'], capture=False) + subp.subp(['rhn-profile-sync', '--verbose'], capture=False) already_registered = True - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code != 1: raise return already_registered @@ -65,7 +65,7 @@ def do_register(server, profile_name, cmd.extend(['--sslCACert', str(ca_cert_path)]) if activation_key: cmd.extend(['--activationkey', str(activation_key)]) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def handle(name, cfg, cloud, log, _args): diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 163cce99..228e5e0d 100755 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -116,6 +116,7 @@ import sys from cloudinit.distros import ug_util from cloudinit import ssh_util +from cloudinit import subp from cloudinit import util @@ -164,7 +165,7 @@ def handle(_name, cfg, cloud, log, _args): try: # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) except Exception: util.logexc(log, "Failed generated a key for %s from %s", @@ -186,9 +187,9 @@ def handle(_name, cfg, cloud, log, _args): # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): try: - out, err = util.subp(cmd, capture=True, env=lang_c) + out, err = subp.subp(cmd, capture=True, env=lang_c) sys.stdout.write(util.decode_binary(out)) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: err = util.decode_binary(e.stderr).lower() if (e.exit_code == 1 and err.lower().startswith("unknown key")): diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 63f87298..856e5a9e 100755 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -31,6 +31,7 @@ either ``lp:`` for launchpad or ``gh:`` for github to the username. """ from cloudinit.distros import ug_util +from cloudinit import subp from cloudinit import util import pwd @@ -101,8 +102,8 @@ def import_ssh_ids(ids, user, log): log.debug("Importing SSH ids for user %s.", user) try: - util.subp(cmd, capture=False) - except util.ProcessExecutionError as exc: + subp.subp(cmd, capture=False) + except subp.ProcessExecutionError as exc: util.logexc(log, "Failed to run command to import %s SSH ids", user) raise exc diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py index 8b6d2a1a..35ded5db 100644 --- a/cloudinit/config/cc_ubuntu_advantage.py +++ b/cloudinit/config/cc_ubuntu_advantage.py @@ -8,6 +8,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util @@ -109,8 +110,8 @@ def configure_ua(token=None, enable=None): attach_cmd = ['ua', 'attach', token] LOG.debug('Attaching to Ubuntu Advantage. %s', ' '.join(attach_cmd)) try: - util.subp(attach_cmd) - except util.ProcessExecutionError as e: + subp.subp(attach_cmd) + except subp.ProcessExecutionError as e: msg = 'Failure attaching Ubuntu Advantage:\n{error}'.format( error=str(e)) util.logexc(LOG, msg) @@ -119,8 +120,8 @@ def configure_ua(token=None, enable=None): for service in enable: try: cmd = ['ua', 'enable', service] - util.subp(cmd, capture=True) - except util.ProcessExecutionError as e: + subp.subp(cmd, capture=True) + except subp.ProcessExecutionError as e: enable_errors.append((service, e)) if enable_errors: for service, error in enable_errors: @@ -135,7 +136,7 @@ def configure_ua(token=None, enable=None): def maybe_install_ua_tools(cloud): """Install ubuntu-advantage-tools if not present.""" - if util.which('ua'): + if subp.which('ua'): return try: cloud.distro.update_package_sources() diff --git a/cloudinit/config/cc_ubuntu_drivers.py b/cloudinit/config/cc_ubuntu_drivers.py index 297451d6..2d1d2b32 100644 --- a/cloudinit/config/cc_ubuntu_drivers.py +++ b/cloudinit/config/cc_ubuntu_drivers.py @@ -9,6 +9,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import temp_utils from cloudinit import type_utils from cloudinit import util @@ -108,7 +109,7 @@ def install_drivers(cfg, pkg_install_func): LOG.debug("Not installing NVIDIA drivers. %s=%s", cfgpath, nv_acc) return - if not util.which('ubuntu-drivers'): + if not subp.which('ubuntu-drivers'): LOG.debug("'ubuntu-drivers' command not available. " "Installing ubuntu-drivers-common") pkg_install_func(['ubuntu-drivers-common']) @@ -131,7 +132,7 @@ def install_drivers(cfg, pkg_install_func): debconf_script, util.encode_text(NVIDIA_DRIVER_LATELINK_DEBCONF_SCRIPT), mode=0o755) - util.subp([debconf_script, debconf_file]) + subp.subp([debconf_script, debconf_file]) except Exception as e: util.logexc( LOG, "Failed to register NVIDIA debconf template: %s", str(e)) @@ -141,8 +142,8 @@ def install_drivers(cfg, pkg_install_func): util.del_dir(tdir) try: - util.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg]) - except util.ProcessExecutionError as exc: + subp.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg]) + except subp.ProcessExecutionError as exc: if OLD_UBUNTU_DRIVERS_STDERR_NEEDLE in exc.stderr: LOG.warning('the available version of ubuntu-drivers is' ' too old to perform requested driver installation') diff --git a/cloudinit/config/tests/test_disable_ec2_metadata.py b/cloudinit/config/tests/test_disable_ec2_metadata.py index 823917c7..b00f2083 100644 --- a/cloudinit/config/tests/test_disable_ec2_metadata.py +++ b/cloudinit/config/tests/test_disable_ec2_metadata.py @@ -15,8 +15,8 @@ DISABLE_CFG = {'disable_ec2_metadata': 'true'} class TestEC2MetadataRoute(CiTestCase): - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp') def test_disable_ifconfig(self, m_subp, m_which): """Set the route if ifconfig command is available""" m_which.side_effect = lambda x: x if x == 'ifconfig' else None @@ -25,8 +25,8 @@ class TestEC2MetadataRoute(CiTestCase): ['route', 'add', '-host', '169.254.169.254', 'reject'], capture=False) - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp') def test_disable_ip(self, m_subp, m_which): """Set the route if ip command is available""" m_which.side_effect = lambda x: x if x == 'ip' else None @@ -35,8 +35,8 @@ class TestEC2MetadataRoute(CiTestCase): ['ip', 'route', 'add', 'prohibit', '169.254.169.254'], capture=False) - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp') def test_disable_no_tool(self, m_subp, m_which): """Log error when neither route nor ip commands are available""" m_which.return_value = None # Find neither ifconfig nor ip diff --git a/cloudinit/config/tests/test_grub_dpkg.py b/cloudinit/config/tests/test_grub_dpkg.py index 01efa330..99c05bb5 100644 --- a/cloudinit/config/tests/test_grub_dpkg.py +++ b/cloudinit/config/tests/test_grub_dpkg.py @@ -4,7 +4,7 @@ import pytest from unittest import mock from logging import Logger -from cloudinit.util import ProcessExecutionError +from cloudinit.subp import ProcessExecutionError from cloudinit.config.cc_grub_dpkg import fetch_idevs, handle @@ -79,7 +79,7 @@ class TestFetchIdevs: ) @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc") @mock.patch("cloudinit.config.cc_grub_dpkg.os.path.exists") - @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp") + @mock.patch("cloudinit.config.cc_grub_dpkg.subp.subp") def test_fetch_idevs(self, m_subp, m_exists, m_logexc, grub_output, path_exists, expected_log_call, udevadm_output, expected_idevs): @@ -158,7 +158,7 @@ class TestHandle: @mock.patch("cloudinit.config.cc_grub_dpkg.fetch_idevs") @mock.patch("cloudinit.config.cc_grub_dpkg.util.get_cfg_option_str") @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc") - @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp") + @mock.patch("cloudinit.config.cc_grub_dpkg.subp.subp") def test_handle(self, m_subp, m_logexc, m_get_cfg_str, m_fetch_idevs, cfg_idevs, cfg_idevs_empty, fetch_idevs_output, expected_log_output): diff --git a/cloudinit/config/tests/test_mounts.py b/cloudinit/config/tests/test_mounts.py index 80b54d0f..764a33e3 100644 --- a/cloudinit/config/tests/test_mounts.py +++ b/cloudinit/config/tests/test_mounts.py @@ -13,12 +13,12 @@ class TestCreateSwapfile: @pytest.mark.parametrize('fstype', ('xfs', 'btrfs', 'ext4', 'other')) @mock.patch(M_PATH + 'util.get_mount_info') - @mock.patch(M_PATH + 'util.subp') + @mock.patch(M_PATH + 'subp.subp') def test_happy_path(self, m_subp, m_get_mount_info, fstype, tmpdir): swap_file = tmpdir.join("swap-file") fname = str(swap_file) - # Some of the calls to util.subp should create the swap file; this + # Some of the calls to subp.subp should create the swap file; this # roughly approximates that m_subp.side_effect = lambda *args, **kwargs: swap_file.write('') diff --git a/cloudinit/config/tests/test_set_passwords.py b/cloudinit/config/tests/test_set_passwords.py index 2732bd60..daa1ef51 100644 --- a/cloudinit/config/tests/test_set_passwords.py +++ b/cloudinit/config/tests/test_set_passwords.py @@ -14,7 +14,7 @@ class TestHandleSshPwauth(CiTestCase): with_logs = True - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_unknown_value_logs_warning(self, m_subp): setpass.handle_ssh_pwauth("floo") self.assertIn("Unrecognized value: ssh_pwauth=floo", @@ -22,7 +22,7 @@ class TestHandleSshPwauth(CiTestCase): m_subp.assert_not_called() @mock.patch(MODPATH + "update_ssh_config", return_value=True) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_systemctl_as_service_cmd(self, m_subp, m_update_ssh_config): """If systemctl in service cmd: systemctl restart name.""" setpass.handle_ssh_pwauth( @@ -31,7 +31,7 @@ class TestHandleSshPwauth(CiTestCase): m_subp.call_args) @mock.patch(MODPATH + "update_ssh_config", return_value=True) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_service_as_service_cmd(self, m_subp, m_update_ssh_config): """If systemctl in service cmd: systemctl restart name.""" setpass.handle_ssh_pwauth( @@ -40,7 +40,7 @@ class TestHandleSshPwauth(CiTestCase): m_subp.call_args) @mock.patch(MODPATH + "update_ssh_config", return_value=False) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_not_restarted_if_not_updated(self, m_subp, m_update_ssh_config): """If config is not updated, then no system restart should be done.""" setpass.handle_ssh_pwauth(True) @@ -48,7 +48,7 @@ class TestHandleSshPwauth(CiTestCase): self.assertIn("No need to restart SSH", self.logs.getvalue()) @mock.patch(MODPATH + "update_ssh_config", return_value=True) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_unchanged_does_nothing(self, m_subp, m_update_ssh_config): """If 'unchanged', then no updates to config and no restart.""" setpass.handle_ssh_pwauth( @@ -56,7 +56,7 @@ class TestHandleSshPwauth(CiTestCase): m_update_ssh_config.assert_not_called() m_subp.assert_not_called() - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_valid_change_values(self, m_subp): """If value is a valid changen value, then update should be called.""" upname = MODPATH + "update_ssh_config" @@ -88,7 +88,7 @@ class TestSetPasswordsHandle(CiTestCase): 'ssh_pwauth=None\n', self.logs.getvalue()) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_handle_on_chpasswd_list_parses_common_hashes(self, m_subp): """handle parses command password hashes.""" cloud = self.tmp_cloud(distro='ubuntu') @@ -98,7 +98,7 @@ class TestSetPasswordsHandle(CiTestCase): 'ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoakMMC7dR52q' 'SDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXazGGx3oo1'] cfg = {'chpasswd': {'list': valid_hashed_pwds}} - with mock.patch(MODPATH + 'util.subp') as m_subp: + with mock.patch(MODPATH + 'subp.subp') as m_subp: setpass.handle( 'IGNORED', cfg=cfg, cloud=cloud, log=self.logger, args=[]) self.assertIn( @@ -113,7 +113,7 @@ class TestSetPasswordsHandle(CiTestCase): m_subp.call_args_list) @mock.patch(MODPATH + "util.is_BSD") - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_bsd_calls_custom_pw_cmds_to_set_and_expire_passwords( self, m_subp, m_is_bsd): """BSD don't use chpasswd""" @@ -130,7 +130,7 @@ class TestSetPasswordsHandle(CiTestCase): m_subp.call_args_list) @mock.patch(MODPATH + "util.is_BSD") - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_handle_on_chpasswd_list_creates_random_passwords(self, m_subp, m_is_bsd): """handle parses command set random passwords.""" @@ -140,7 +140,7 @@ class TestSetPasswordsHandle(CiTestCase): 'root:R', 'ubuntu:RANDOM'] cfg = {'chpasswd': {'expire': 'false', 'list': valid_random_pwds}} - with mock.patch(MODPATH + 'util.subp') as m_subp: + with mock.patch(MODPATH + 'subp.subp') as m_subp: setpass.handle( 'IGNORED', cfg=cfg, cloud=cloud, log=self.logger, args=[]) self.assertIn( diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py index 95270fa0..6d4c014a 100644 --- a/cloudinit/config/tests/test_snap.py +++ b/cloudinit/config/tests/test_snap.py @@ -92,7 +92,7 @@ class TestAddAssertions(CiTestCase): super(TestAddAssertions, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_add_assertions_on_empty_list(self, m_subp): """When provided with an empty list, add_assertions does nothing.""" add_assertions([]) @@ -107,7 +107,7 @@ class TestAddAssertions(CiTestCase): "assertion parameter was not a list or dict: I'm Not Valid", str(context_manager.exception)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_add_assertions_adds_assertions_as_list(self, m_subp): """When provided with a list, add_assertions adds all assertions.""" self.assertEqual( @@ -130,7 +130,7 @@ class TestAddAssertions(CiTestCase): self.assertEqual( util.load_file(compare_file), util.load_file(assert_file)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_add_assertions_adds_assertions_as_dict(self, m_subp): """When provided with a dict, add_assertions adds all assertions.""" self.assertEqual( @@ -168,7 +168,7 @@ class TestRunCommands(CiTestCase): super(TestRunCommands, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_run_commands_on_empty_list(self, m_subp): """When provided with an empty list, run_commands does nothing.""" run_commands([]) @@ -477,7 +477,7 @@ class TestHandle(CiTestCase): self.assertEqual('HI\nMOM\n', util.load_file(outfile)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_handle_adds_assertions(self, m_subp): """Any configured snap assertions are provided to add_assertions.""" assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) @@ -493,7 +493,7 @@ class TestHandle(CiTestCase): self.assertEqual( util.load_file(compare_file), util.load_file(assert_file)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') @skipUnlessJsonSchema() def test_handle_validates_schema(self, m_subp): """Any provided configuration is runs validate_cloudconfig_schema.""" diff --git a/cloudinit/config/tests/test_ubuntu_advantage.py b/cloudinit/config/tests/test_ubuntu_advantage.py index 8c4161ef..db7fb726 100644 --- a/cloudinit/config/tests/test_ubuntu_advantage.py +++ b/cloudinit/config/tests/test_ubuntu_advantage.py @@ -3,7 +3,7 @@ from cloudinit.config.cc_ubuntu_advantage import ( configure_ua, handle, maybe_install_ua_tools, schema) from cloudinit.config.schema import validate_cloudconfig_schema -from cloudinit import util +from cloudinit import subp from cloudinit.tests.helpers import ( CiTestCase, mock, SchemaTestCaseMixin, skipUnlessJsonSchema) @@ -26,10 +26,10 @@ class TestConfigureUA(CiTestCase): super(TestConfigureUA, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_error(self, m_subp): """Errors from ua attach command are raised.""" - m_subp.side_effect = util.ProcessExecutionError( + m_subp.side_effect = subp.ProcessExecutionError( 'Invalid token SomeToken') with self.assertRaises(RuntimeError) as context_manager: configure_ua(token='SomeToken') @@ -39,7 +39,7 @@ class TestConfigureUA(CiTestCase): 'Stdout: Invalid token SomeToken\nStderr: -', str(context_manager.exception)) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_token(self, m_subp): """When token is provided, attach the machine to ua using the token.""" configure_ua(token='SomeToken') @@ -48,7 +48,7 @@ class TestConfigureUA(CiTestCase): 'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n', self.logs.getvalue()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_on_service_error(self, m_subp): """all services should be enabled and then any failures raised""" @@ -56,7 +56,7 @@ class TestConfigureUA(CiTestCase): fail_cmds = [['ua', 'enable', svc] for svc in ['esm', 'cc']] if cmd in fail_cmds and capture: svc = cmd[-1] - raise util.ProcessExecutionError( + raise subp.ProcessExecutionError( 'Invalid {} credentials'.format(svc.upper())) m_subp.side_effect = fake_subp @@ -83,7 +83,7 @@ class TestConfigureUA(CiTestCase): 'Failure enabling Ubuntu Advantage service(s): "esm", "cc"', str(context_manager.exception)) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_empty_services(self, m_subp): """When services is an empty list, do not auto-enable attach.""" configure_ua(token='SomeToken', enable=[]) @@ -92,7 +92,7 @@ class TestConfigureUA(CiTestCase): 'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n', self.logs.getvalue()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_specific_services(self, m_subp): """When services a list, only enable specific services.""" configure_ua(token='SomeToken', enable=['fips']) @@ -105,7 +105,7 @@ class TestConfigureUA(CiTestCase): self.logs.getvalue()) @mock.patch('%s.maybe_install_ua_tools' % MPATH, mock.MagicMock()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_string_services(self, m_subp): """When services a string, treat as singleton list and warn""" configure_ua(token='SomeToken', enable='fips') @@ -119,7 +119,7 @@ class TestConfigureUA(CiTestCase): 'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n', self.logs.getvalue()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_weird_services(self, m_subp): """When services not string or list, warn but still attach""" configure_ua(token='SomeToken', enable={'deffo': 'wont work'}) @@ -285,7 +285,7 @@ class TestMaybeInstallUATools(CiTestCase): super(TestMaybeInstallUATools, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_tools_noop_when_ua_tools_present(self, m_which): """Do nothing if ubuntu-advantage-tools already exists.""" m_which.return_value = '/usr/bin/ua' # already installed @@ -294,7 +294,7 @@ class TestMaybeInstallUATools(CiTestCase): 'Some apt error') maybe_install_ua_tools(cloud=FakeCloud(distro)) # No RuntimeError - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_tools_raises_update_errors(self, m_which): """maybe_install_ua_tools logs and raises apt update errors.""" m_which.return_value = None @@ -306,7 +306,7 @@ class TestMaybeInstallUATools(CiTestCase): self.assertEqual('Some apt error', str(context_manager.exception)) self.assertIn('Package update failed\nTraceback', self.logs.getvalue()) - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_raises_install_errors(self, m_which): """maybe_install_ua_tools logs and raises package install errors.""" m_which.return_value = None @@ -320,7 +320,7 @@ class TestMaybeInstallUATools(CiTestCase): self.assertIn( 'Failed to install ubuntu-advantage-tools\n', self.logs.getvalue()) - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_tools_happy_path(self, m_which): """maybe_install_ua_tools installs ubuntu-advantage-tools.""" m_which.return_value = None diff --git a/cloudinit/config/tests/test_ubuntu_drivers.py b/cloudinit/config/tests/test_ubuntu_drivers.py index 0aec1265..504ba356 100644 --- a/cloudinit/config/tests/test_ubuntu_drivers.py +++ b/cloudinit/config/tests/test_ubuntu_drivers.py @@ -7,7 +7,7 @@ from cloudinit.tests.helpers import CiTestCase, skipUnlessJsonSchema, mock from cloudinit.config.schema import ( SchemaValidationError, validate_cloudconfig_schema) from cloudinit.config import cc_ubuntu_drivers as drivers -from cloudinit.util import ProcessExecutionError +from cloudinit.subp import ProcessExecutionError MPATH = "cloudinit.config.cc_ubuntu_drivers." M_TMP_PATH = MPATH + "temp_utils.mkdtemp" @@ -53,8 +53,8 @@ class TestUbuntuDrivers(CiTestCase): schema=drivers.schema, strict=True) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=False) def _assert_happy_path_taken( self, config, m_which, m_subp, m_tmp): """Positive path test through handle. Package should be installed.""" @@ -80,8 +80,8 @@ class TestUbuntuDrivers(CiTestCase): self._assert_happy_path_taken(new_config) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp") - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp") + @mock.patch(MPATH + "subp.which", return_value=False) def test_handle_raises_error_if_no_drivers_found( self, m_which, m_subp, m_tmp): """If ubuntu-drivers doesn't install any drivers, raise an error.""" @@ -109,8 +109,8 @@ class TestUbuntuDrivers(CiTestCase): self.assertIn('ubuntu-drivers found no drivers for installation', self.logs.getvalue()) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=False) def _assert_inert_with_config(self, config, m_which, m_subp): """Helper to reduce repetition when testing negative cases""" myCloud = mock.MagicMock() @@ -154,8 +154,8 @@ class TestUbuntuDrivers(CiTestCase): self.assertEqual(0, m_install_drivers.call_count) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=True) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=True) def test_install_drivers_no_install_if_present( self, m_which, m_subp, m_tmp): """If 'ubuntu-drivers' is present, no package install should occur.""" @@ -181,8 +181,8 @@ class TestUbuntuDrivers(CiTestCase): self.assertEqual(0, pkg_install.call_count) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp") - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp") + @mock.patch(MPATH + "subp.which", return_value=False) def test_install_drivers_handles_old_ubuntu_drivers_gracefully( self, m_which, m_subp, m_tmp): """Older ubuntu-drivers versions should emit message and raise error""" @@ -219,8 +219,8 @@ class TestUbuntuDriversWithVersion(TestUbuntuDrivers): install_gpgpu = ['ubuntu-drivers', 'install', '--gpgpu', 'nvidia:123'] @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=False) def test_version_none_uses_latest(self, m_which, m_subp, m_tmp): tdir = self.tmp_dir() debconf_file = os.path.join(tdir, 'nvidia.template') diff --git a/cloudinit/conftest.py b/cloudinit/conftest.py index 37cbbcda..251bca59 100644 --- a/cloudinit/conftest.py +++ b/cloudinit/conftest.py @@ -2,32 +2,32 @@ from unittest import mock import pytest -from cloudinit import util +from cloudinit import subp @pytest.yield_fixture(autouse=True) def disable_subp_usage(request): """ - Across all (pytest) tests, ensure that util.subp is not invoked. + Across all (pytest) tests, ensure that subp.subp is not invoked. Note that this can only catch invocations where the util module is imported - and ``util.subp(...)`` is called. ``from cloudinit.util import subp`` + and ``subp.subp(...)`` is called. ``from cloudinit.subp mport subp`` imports happen before the patching here (or the CiTestCase monkey-patching) happens, so are left untouched. - To allow a particular test method or class to use util.subp you can set the + To allow a particular test method or class to use subp.subp you can set the parameter passed to this fixture to False using pytest.mark.parametrize:: @pytest.mark.parametrize("disable_subp_usage", [False], indirect=True) def test_whoami(self): - util.subp(["whoami"]) + subp.subp(["whoami"]) - To instead allow util.subp usage for a specific command, you can set the + To instead allow subp.subp usage for a specific command, you can set the parameter passed to this fixture to that command: @pytest.mark.parametrize("disable_subp_usage", ["bash"], indirect=True) def test_bash(self): - util.subp(["bash"]) + subp.subp(["bash"]) To specify multiple commands, set the parameter to a list (note the double-layered list: we specify a single parameter that is itself a list): @@ -35,8 +35,8 @@ def disable_subp_usage(request): @pytest.mark.parametrize( "disable_subp_usage", ["bash", "whoami"], indirect=True) def test_several_things(self): - util.subp(["bash"]) - util.subp(["whoami"]) + subp.subp(["bash"]) + subp.subp(["whoami"]) This fixture (roughly) mirrors the functionality of CiTestCase.allowed_subp. N.B. While autouse fixtures do affect non-pytest @@ -47,11 +47,11 @@ def disable_subp_usage(request): if should_disable: if not isinstance(should_disable, (list, str)): def side_effect(args, *other_args, **kwargs): - raise AssertionError("Unexpectedly used util.subp") + raise AssertionError("Unexpectedly used subp.subp") else: # Look this up before our patch is in place, so we have access to # the real implementation in side_effect - subp = util.subp + real_subp = subp.subp if isinstance(should_disable, str): should_disable = [should_disable] @@ -60,12 +60,12 @@ def disable_subp_usage(request): cmd = args[0] if cmd not in should_disable: raise AssertionError( - "Unexpectedly used util.subp to call {} (allowed:" + "Unexpectedly used subp.subp to call {} (allowed:" " {})".format(cmd, ",".join(should_disable)) ) - return subp(args, *other_args, **kwargs) + return real_subp(args, *other_args, **kwargs) - with mock.patch('cloudinit.util.subp', autospec=True) as m_subp: + with mock.patch('cloudinit.subp.subp', autospec=True) as m_subp: m_subp.side_effect = side_effect yield else: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 35a10590..016ba64d 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -25,6 +25,7 @@ from cloudinit.net import network_state from cloudinit.net import renderers from cloudinit import ssh_util from cloudinit import type_utils +from cloudinit import subp from cloudinit import util from cloudinit.distros.parsers import hosts @@ -225,8 +226,8 @@ class Distro(metaclass=abc.ABCMeta): LOG.debug("Non-persistently setting the system hostname to %s", hostname) try: - util.subp(['hostname', hostname]) - except util.ProcessExecutionError: + subp.subp(['hostname', hostname]) + except subp.ProcessExecutionError: util.logexc(LOG, "Failed to non-persistently adjust the system " "hostname to %s", hostname) @@ -361,12 +362,12 @@ class Distro(metaclass=abc.ABCMeta): LOG.debug("Attempting to run bring up interface %s using command %s", device_name, cmd) try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False @@ -480,7 +481,7 @@ class Distro(metaclass=abc.ABCMeta): # Run the command LOG.debug("Adding user %s", name) try: - util.subp(useradd_cmd, logstring=log_useradd_cmd) + subp.subp(useradd_cmd, logstring=log_useradd_cmd) except Exception as e: util.logexc(LOG, "Failed to create user %s", name) raise e @@ -500,7 +501,7 @@ class Distro(metaclass=abc.ABCMeta): # Run the command LOG.debug("Adding snap user %s", name) try: - (out, err) = util.subp(create_user_cmd, logstring=create_user_cmd, + (out, err) = subp.subp(create_user_cmd, logstring=create_user_cmd, capture=True) LOG.debug("snap create-user returned: %s:%s", out, err) jobj = util.load_json(out) @@ -582,20 +583,20 @@ class Distro(metaclass=abc.ABCMeta): # passwd must use short '-l' due to SLES11 lacking long form '--lock' lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name]) try: - cmd = next(tool for tool in lock_tools if util.which(tool[0])) + cmd = next(tool for tool in lock_tools if subp.which(tool[0])) except StopIteration: raise RuntimeError(( "Unable to lock user account '%s'. No tools available. " " Tried: %s.") % (name, [c[0] for c in lock_tools])) try: - util.subp(cmd) + subp.subp(cmd) except Exception as e: util.logexc(LOG, 'Failed to disable password for user %s', name) raise e def expire_passwd(self, user): try: - util.subp(['passwd', '--expire', user]) + subp.subp(['passwd', '--expire', user]) except Exception as e: util.logexc(LOG, "Failed to set 'expire' for %s", user) raise e @@ -611,7 +612,7 @@ class Distro(metaclass=abc.ABCMeta): cmd.append('-e') try: - util.subp(cmd, pass_string, logstring="chpasswd for %s" % user) + subp.subp(cmd, pass_string, logstring="chpasswd for %s" % user) except Exception as e: util.logexc(LOG, "Failed to set password for %s", user) raise e @@ -708,7 +709,7 @@ class Distro(metaclass=abc.ABCMeta): LOG.warning("Skipping creation of existing group '%s'", name) else: try: - util.subp(group_add_cmd) + subp.subp(group_add_cmd) LOG.info("Created new group %s", name) except Exception: util.logexc(LOG, "Failed to create group %s", name) @@ -721,7 +722,7 @@ class Distro(metaclass=abc.ABCMeta): "; user does not exist.", member, name) continue - util.subp(['usermod', '-a', '-G', name, member]) + subp.subp(['usermod', '-a', '-G', name, member]) LOG.info("Added user '%s' to group '%s'", member, name) diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 9f89c5f9..038aa9ac 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -8,6 +8,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging from cloudinit import util +from cloudinit import subp from cloudinit.distros import net_util from cloudinit.distros.parsers.hostname import HostnameConf @@ -44,7 +45,7 @@ class Distro(distros.Distro): def apply_locale(self, locale, out_fn=None): if not out_fn: out_fn = self.locale_conf_fn - util.subp(['locale-gen', '-G', locale], capture=False) + subp.subp(['locale-gen', '-G', locale], capture=False) # "" provides trailing newline during join lines = [ util.make_header(), @@ -76,11 +77,11 @@ class Distro(distros.Distro): def _enable_interface(self, device_name): cmd = ['netctl', 'reenable', device_name] try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) def _bring_up_interface(self, device_name): @@ -88,12 +89,12 @@ class Distro(distros.Distro): LOG.debug("Attempting to run bring up interface %s using command %s", device_name, cmd) try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False @@ -158,7 +159,7 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def update_package_sources(self): self._runner.run("update-sources", self.package_command, @@ -173,8 +174,8 @@ def _render_network(entries, target="/", conf_dir="etc/netctl", devs = [] nameservers = [] - resolv_conf = util.target_path(target, resolv_conf) - conf_dir = util.target_path(target, conf_dir) + resolv_conf = subp.target_path(target, resolv_conf) + conf_dir = subp.target_path(target, conf_dir) for (dev, info) in entries.items(): if dev == 'lo': diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py index 37cf93bf..c2d1f77d 100644 --- a/cloudinit/distros/bsd.py +++ b/cloudinit/distros/bsd.py @@ -5,6 +5,7 @@ from cloudinit.distros import bsd_utils from cloudinit import helpers from cloudinit import log as logging from cloudinit import net +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -50,7 +51,7 @@ class BSD(distros.Distro): else: group_add_cmd = self.group_add_cmd_prefix + [name] try: - util.subp(group_add_cmd) + subp.subp(group_add_cmd) LOG.info("Created new group %s", name) except Exception: util.logexc(LOG, "Failed to create group %s", name) @@ -63,7 +64,7 @@ class BSD(distros.Distro): "; user does not exist.", member, name) continue try: - util.subp(self._get_add_member_to_group_cmd(member, name)) + subp.subp(self._get_add_member_to_group_cmd(member, name)) LOG.info("Added user '%s' to group '%s'", member, name) except Exception: util.logexc(LOG, "Failed to add user '%s' to group '%s'", @@ -111,7 +112,7 @@ class BSD(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, env=self._get_pkg_cmd_environ(), capture=False) + subp.subp(cmd, env=self._get_pkg_cmd_environ(), capture=False) def _write_network_config(self, netconfig): return self._supported_write_network_config(netconfig) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 128bb523..844aaf21 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -13,6 +13,7 @@ import os from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros.parsers.hostname import HostnameConf @@ -197,7 +198,7 @@ class Distro(distros.Distro): # Allow the output of this to flow outwards (ie not be captured) util.log_time(logfunc=LOG.debug, msg="apt-%s [%s]" % (command, ' '.join(cmd)), - func=util.subp, + func=subp.subp, args=(cmd,), kwargs={'env': e, 'capture': False}) def update_package_sources(self): @@ -214,7 +215,7 @@ def _get_wrapper_prefix(cmd, mode): if (util.is_true(mode) or (str(mode).lower() == "auto" and cmd[0] and - util.which(cmd[0]))): + subp.which(cmd[0]))): return cmd else: return [] @@ -269,7 +270,7 @@ def update_locale_conf(locale, sys_path, keyname='LANG'): """Update system locale config""" LOG.debug('Updating %s with locale setting %s=%s', sys_path, keyname, locale) - util.subp( + subp.subp( ['update-locale', '--locale-file=' + sys_path, '%s=%s' % (keyname, locale)], capture=False) @@ -291,7 +292,7 @@ def regenerate_locale(locale, sys_path, keyname='LANG'): # finally, trigger regeneration LOG.debug('Generating locales for %s', locale) - util.subp(['locale-gen', locale], capture=False) + subp.subp(['locale-gen', locale], capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index b3a4ad67..dde34d41 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -10,6 +10,7 @@ from io import StringIO import cloudinit.distros.bsd from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -78,7 +79,7 @@ class Distro(cloudinit.distros.bsd.BSD): # Run the command LOG.info("Adding user %s", name) try: - util.subp(pw_useradd_cmd, logstring=log_pw_useradd_cmd) + subp.subp(pw_useradd_cmd, logstring=log_pw_useradd_cmd) except Exception: util.logexc(LOG, "Failed to create user %s", name) raise @@ -90,7 +91,7 @@ class Distro(cloudinit.distros.bsd.BSD): def expire_passwd(self, user): try: - util.subp(['pw', 'usermod', user, '-p', '01-Jan-1970']) + subp.subp(['pw', 'usermod', user, '-p', '01-Jan-1970']) except Exception: util.logexc(LOG, "Failed to set pw expiration for %s", user) raise @@ -102,7 +103,7 @@ class Distro(cloudinit.distros.bsd.BSD): hash_opt = "-h" try: - util.subp(['pw', 'usermod', user, hash_opt, '0'], + subp.subp(['pw', 'usermod', user, hash_opt, '0'], data=passwd, logstring="chpasswd for %s" % user) except Exception: util.logexc(LOG, "Failed to set password for %s", user) @@ -110,7 +111,7 @@ class Distro(cloudinit.distros.bsd.BSD): def lock_passwd(self, name): try: - util.subp(['pw', 'usermod', name, '-h', '-']) + subp.subp(['pw', 'usermod', name, '-h', '-']) except Exception: util.logexc(LOG, "Failed to lock user %s", name) raise @@ -131,8 +132,8 @@ class Distro(cloudinit.distros.bsd.BSD): try: LOG.debug("Running cap_mkdb for %s", locale) - util.subp(['cap_mkdb', self.login_conf_fn]) - except util.ProcessExecutionError: + subp.subp(['cap_mkdb', self.login_conf_fn]) + except subp.ProcessExecutionError: # cap_mkdb failed, so restore the backup. util.logexc(LOG, "Failed to apply locale %s", locale) try: diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index dc57717d..2bee1c89 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -9,6 +9,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros import net_util @@ -39,7 +40,7 @@ class Distro(distros.Distro): def apply_locale(self, locale, out_fn=None): if not out_fn: out_fn = self.locale_conf_fn - util.subp(['locale-gen', '-G', locale], capture=False) + subp.subp(['locale-gen', '-G', locale], capture=False) # "" provides trailing newline during join lines = [ util.make_header(), @@ -94,11 +95,11 @@ class Distro(distros.Distro): cmd = ['rc-update', 'add', 'net.{name}'.format(name=dev), 'default'] try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) @@ -119,12 +120,12 @@ class Distro(distros.Distro): LOG.debug("Attempting to run bring up interface %s using command %s", device_name, cmd) try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False @@ -137,11 +138,11 @@ class Distro(distros.Distro): # Grab device names from init scripts cmd = ['ls', '/etc/init.d/net.*'] try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False devices = [x.split('.')[2] for x in _out.split(' ')] @@ -208,7 +209,7 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def update_package_sources(self): self._runner.run("update-sources", self.package_command, diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py index ecc8239a..066737a8 100644 --- a/cloudinit/distros/netbsd.py +++ b/cloudinit/distros/netbsd.py @@ -8,6 +8,7 @@ import platform import cloudinit.distros.bsd from cloudinit import log as logging +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -76,7 +77,7 @@ class NetBSD(cloudinit.distros.bsd.BSD): # Run the command LOG.info("Adding user %s", name) try: - util.subp(adduser_cmd, logstring=log_adduser_cmd) + subp.subp(adduser_cmd, logstring=log_adduser_cmd) except Exception: util.logexc(LOG, "Failed to create user %s", name) raise @@ -103,7 +104,7 @@ class NetBSD(cloudinit.distros.bsd.BSD): crypt.mksalt(method)) try: - util.subp(['usermod', '-p', hashed_pw, user]) + subp.subp(['usermod', '-p', hashed_pw, user]) except Exception: util.logexc(LOG, "Failed to set password for %s", user) raise @@ -111,21 +112,21 @@ class NetBSD(cloudinit.distros.bsd.BSD): def force_passwd_change(self, user): try: - util.subp(['usermod', '-F', user]) + subp.subp(['usermod', '-F', user]) except Exception: util.logexc(LOG, "Failed to set pw expiration for %s", user) raise def lock_passwd(self, name): try: - util.subp(['usermod', '-C', 'yes', name]) + subp.subp(['usermod', '-C', 'yes', name]) except Exception: util.logexc(LOG, "Failed to lock user %s", name) raise def unlock_passwd(self, name): try: - util.subp(['usermod', '-C', 'no', name]) + subp.subp(['usermod', '-C', 'no', name]) except Exception: util.logexc(LOG, "Failed to unlock user %s", name) raise diff --git a/cloudinit/distros/openbsd.py b/cloudinit/distros/openbsd.py index ca094156..07c76530 100644 --- a/cloudinit/distros/openbsd.py +++ b/cloudinit/distros/openbsd.py @@ -7,6 +7,7 @@ import platform import cloudinit.distros.netbsd from cloudinit import log as logging +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -27,7 +28,7 @@ class Distro(cloudinit.distros.netbsd.NetBSD): def lock_passwd(self, name): try: - util.subp(['usermod', '-p', '*', name]) + subp.subp(['usermod', '-p', '*', name]) except Exception: util.logexc(LOG, "Failed to lock user %s", name) raise diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py index 68028d20..ffb7d0e8 100644 --- a/cloudinit/distros/opensuse.py +++ b/cloudinit/distros/opensuse.py @@ -14,6 +14,7 @@ from cloudinit.distros.parsers.hostname import HostnameConf from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros import rhel_util as rhutil @@ -97,7 +98,7 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def set_timezone(self, tz): tz_file = self._find_tz_file(tz) @@ -129,7 +130,7 @@ class Distro(distros.Distro): if self.uses_systemd() and filename.endswith('/previous-hostname'): return util.load_file(filename).strip() elif self.uses_systemd(): - (out, _err) = util.subp(['hostname']) + (out, _err) = subp.subp(['hostname']) if len(out): return out else: @@ -163,7 +164,7 @@ class Distro(distros.Distro): if self.uses_systemd() and out_fn.endswith('/previous-hostname'): util.write_file(out_fn, hostname) elif self.uses_systemd(): - util.subp(['hostnamectl', 'set-hostname', str(hostname)]) + subp.subp(['hostnamectl', 'set-hostname', str(hostname)]) else: conf = None try: diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index f55d96f7..c72f7c17 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -11,6 +11,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros import rhel_util @@ -83,7 +84,7 @@ class Distro(distros.Distro): if self.uses_systemd() and out_fn.endswith('/previous-hostname'): util.write_file(out_fn, hostname) elif self.uses_systemd(): - util.subp(['hostnamectl', 'set-hostname', str(hostname)]) + subp.subp(['hostnamectl', 'set-hostname', str(hostname)]) else: host_cfg = { 'HOSTNAME': hostname, @@ -108,7 +109,7 @@ class Distro(distros.Distro): if self.uses_systemd() and filename.endswith('/previous-hostname'): return util.load_file(filename).strip() elif self.uses_systemd(): - (out, _err) = util.subp(['hostname']) + (out, _err) = subp.subp(['hostname']) if len(out): return out else: @@ -146,7 +147,7 @@ class Distro(distros.Distro): if pkgs is None: pkgs = [] - if util.which('dnf'): + if subp.which('dnf'): LOG.debug('Using DNF for package management') cmd = ['dnf'] else: @@ -173,7 +174,7 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def update_package_sources(self): self._runner.run("update-sources", self.package_command, diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py index 7fe17a2e..72b5ac59 100644 --- a/cloudinit/gpg.py +++ b/cloudinit/gpg.py @@ -8,7 +8,7 @@ """gpg.py - Collection of gpg key related functions""" from cloudinit import log as logging -from cloudinit import util +from cloudinit import subp import time @@ -18,9 +18,9 @@ LOG = logging.getLogger(__name__) def export_armour(key): """Export gpg key, armoured key gets returned""" try: - (armour, _) = util.subp(["gpg", "--export", "--armour", key], + (armour, _) = subp.subp(["gpg", "--export", "--armour", key], capture=True) - except util.ProcessExecutionError as error: + except subp.ProcessExecutionError as error: # debug, since it happens for any key not on the system initially LOG.debug('Failed to export armoured key "%s": %s', key, error) armour = None @@ -51,11 +51,11 @@ def recv_key(key, keyserver, retries=(1, 1)): while True: trynum += 1 try: - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) LOG.debug("Imported key '%s' from keyserver '%s' on try %d", key, keyserver, trynum) return - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: error = e try: naplen = next(sleeps) @@ -72,9 +72,9 @@ def recv_key(key, keyserver, retries=(1, 1)): def delete_key(key): """Delete the specified key from the local gpg ring""" try: - util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], + subp.subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=True) - except util.ProcessExecutionError as error: + except subp.ProcessExecutionError as error: LOG.warning('Failed delete key "%s": %s', key, error) diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index dca50a49..c6205097 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -12,6 +12,7 @@ import os from cloudinit import handlers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.settings import (PER_ALWAYS) @@ -48,8 +49,8 @@ class BootHookPartHandler(handlers.Handler): env = os.environ.copy() if self.instance_id is not None: env['INSTANCE_ID'] = str(self.instance_id) - util.subp([filepath], env=env) - except util.ProcessExecutionError: + subp.subp([filepath], env=env) + except subp.ProcessExecutionError: util.logexc(LOG, "Boothooks script %s execution error", filepath) except Exception: util.logexc(LOG, "Boothooks unknown error when running %s", diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index 003cad60..a9d29537 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -13,6 +13,7 @@ import re from cloudinit import handlers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.settings import (PER_INSTANCE) @@ -52,7 +53,7 @@ class UpstartJobPartHandler(handlers.Handler): util.write_file(path, payload, 0o644) if SUITABLE_UPSTART: - util.subp(["initctl", "reload-configuration"], capture=False) + subp.subp(["initctl", "reload-configuration"], capture=False) def _has_suitable_upstart(): @@ -63,7 +64,7 @@ def _has_suitable_upstart(): if not os.path.exists("/sbin/initctl"): return False try: - (version_out, _err) = util.subp(["initctl", "version"]) + (version_out, _err) = subp.subp(["initctl", "version"]) except Exception: util.logexc(LOG, "initctl version failed") return False @@ -77,7 +78,7 @@ def _has_suitable_upstart(): if not os.path.exists("/usr/bin/dpkg-query"): return False try: - (dpkg_ver, _err) = util.subp(["dpkg-query", + (dpkg_ver, _err) = subp.subp(["dpkg-query", "--showformat=${Version}", "--show", "upstart"], rcs=[0, 1]) except Exception: @@ -86,9 +87,9 @@ def _has_suitable_upstart(): try: good = "1.8-0ubuntu1.2" - util.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good]) + subp.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good]) return True - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code == 1: pass else: diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 8af24fa9..a57fea0a 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -12,6 +12,7 @@ import os import re from functools import partial +from cloudinit import subp from cloudinit import util from cloudinit.net.network_state import mask_to_net_prefix from cloudinit.url_helper import UrlError, readurl @@ -358,7 +359,7 @@ def find_fallback_nic_on_freebsd(blacklist_drivers=None): we'll use the first interface from ``ifconfig -l -u ether`` """ - stdout, _stderr = util.subp(['ifconfig', '-l', '-u', 'ether']) + stdout, _stderr = subp.subp(['ifconfig', '-l', '-u', 'ether']) values = stdout.split() if values: return values[0] @@ -620,9 +621,9 @@ def _get_current_rename_info(check_downable=True): if check_downable: nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]") - ipv6, _err = util.subp(['ip', '-6', 'addr', 'show', 'permanent', + ipv6, _err = subp.subp(['ip', '-6', 'addr', 'show', 'permanent', 'scope', 'global'], capture=True) - ipv4, _err = util.subp(['ip', '-4', 'addr', 'show'], capture=True) + ipv4, _err = subp.subp(['ip', '-4', 'addr', 'show'], capture=True) nics_with_addresses = set() for bytes_out in (ipv6, ipv4): @@ -658,13 +659,13 @@ def _rename_interfaces(renames, strict_present=True, strict_busy=True, for data in cur_info.values()) def rename(cur, new): - util.subp(["ip", "link", "set", cur, "name", new], capture=True) + subp.subp(["ip", "link", "set", cur, "name", new], capture=True) def down(name): - util.subp(["ip", "link", "set", name, "down"], capture=True) + subp.subp(["ip", "link", "set", name, "down"], capture=True) def up(name): - util.subp(["ip", "link", "set", name, "up"], capture=True) + subp.subp(["ip", "link", "set", name, "up"], capture=True) ops = [] errors = [] @@ -819,7 +820,7 @@ def get_interfaces_by_mac(): def get_interfaces_by_mac_on_freebsd(): - (out, _) = util.subp(['ifconfig', '-a', 'ether']) + (out, _) = subp.subp(['ifconfig', '-a', 'ether']) # flatten each interface block in a single line def flatten(out): @@ -850,7 +851,7 @@ def get_interfaces_by_mac_on_netbsd(): re_field_match = ( r"(?P<ifname>\w+).*address:\s" r"(?P<mac>([\da-f]{2}[:-]){5}([\da-f]{2})).*") - (out, _) = util.subp(['ifconfig', '-a']) + (out, _) = subp.subp(['ifconfig', '-a']) if_lines = re.sub(r'\n\s+', ' ', out).splitlines() for line in if_lines: m = re.match(re_field_match, line) @@ -865,7 +866,7 @@ def get_interfaces_by_mac_on_openbsd(): re_field_match = ( r"(?P<ifname>\w+).*lladdr\s" r"(?P<mac>([\da-f]{2}[:-]){5}([\da-f]{2})).*") - (out, _) = util.subp(['ifconfig', '-a']) + (out, _) = subp.subp(['ifconfig', '-a']) if_lines = re.sub(r'\n\s+', ' ', out).splitlines() for line in if_lines: m = re.match(re_field_match, line) @@ -1067,11 +1068,11 @@ class EphemeralIPv4Network(object): def __exit__(self, excp_type, excp_value, excp_traceback): """Teardown anything we set up.""" for cmd in self.cleanup_cmds: - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def _delete_address(self, address, prefix): """Perform the ip command to remove the specified address.""" - util.subp( + subp.subp( ['ip', '-family', 'inet', 'addr', 'del', '%s/%s' % (address, prefix), 'dev', self.interface], capture=True) @@ -1083,11 +1084,11 @@ class EphemeralIPv4Network(object): 'Attempting setup of ephemeral network on %s with %s brd %s', self.interface, cidr, self.broadcast) try: - util.subp( + subp.subp( ['ip', '-family', 'inet', 'addr', 'add', cidr, 'broadcast', self.broadcast, 'dev', self.interface], capture=True, update_env={'LANG': 'C'}) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if "File exists" not in e.stderr: raise LOG.debug( @@ -1095,7 +1096,7 @@ class EphemeralIPv4Network(object): self.interface, self.ip) else: # Address creation success, bring up device and queue cleanup - util.subp( + subp.subp( ['ip', '-family', 'inet', 'link', 'set', 'dev', self.interface, 'up'], capture=True) self.cleanup_cmds.append( @@ -1112,7 +1113,7 @@ class EphemeralIPv4Network(object): via_arg = [] if gateway != "0.0.0.0/0": via_arg = ['via', gateway] - util.subp( + subp.subp( ['ip', '-4', 'route', 'add', net_address] + via_arg + ['dev', self.interface], capture=True) self.cleanup_cmds.insert( @@ -1122,20 +1123,20 @@ class EphemeralIPv4Network(object): def _bringup_router(self): """Perform the ip commands to fully setup the router if needed.""" # Check if a default route exists and exit if it does - out, _ = util.subp(['ip', 'route', 'show', '0.0.0.0/0'], capture=True) + out, _ = subp.subp(['ip', 'route', 'show', '0.0.0.0/0'], capture=True) if 'default' in out: LOG.debug( 'Skip ephemeral route setup. %s already has default route: %s', self.interface, out.strip()) return - util.subp( + subp.subp( ['ip', '-4', 'route', 'add', self.router, 'dev', self.interface, 'src', self.ip], capture=True) self.cleanup_cmds.insert( 0, ['ip', '-4', 'route', 'del', self.router, 'dev', self.interface, 'src', self.ip]) - util.subp( + subp.subp( ['ip', '-4', 'route', 'add', 'default', 'via', self.router, 'dev', self.interface], capture=True) self.cleanup_cmds.insert( diff --git a/cloudinit/net/bsd.py b/cloudinit/net/bsd.py index fb714d4c..1c355a98 100644 --- a/cloudinit/net/bsd.py +++ b/cloudinit/net/bsd.py @@ -5,6 +5,7 @@ import re from cloudinit import log as logging from cloudinit import net from cloudinit import util +from cloudinit import subp from cloudinit.distros.parsers.resolv_conf import ResolvConf from cloudinit.distros import bsd_utils @@ -18,11 +19,11 @@ class BSDRenderer(renderer.Renderer): rc_conf_fn = 'etc/rc.conf' def get_rc_config_value(self, key): - fn = util.target_path(self.target, self.rc_conf_fn) + fn = subp.target_path(self.target, self.rc_conf_fn) bsd_utils.get_rc_config_value(key, fn=fn) def set_rc_config_value(self, key, value): - fn = util.target_path(self.target, self.rc_conf_fn) + fn = subp.target_path(self.target, self.rc_conf_fn) bsd_utils.set_rc_config_value(key, value, fn=fn) def __init__(self, config=None): @@ -111,12 +112,12 @@ class BSDRenderer(renderer.Renderer): # Try to read the /etc/resolv.conf or just start from scratch if that # fails. try: - resolvconf = ResolvConf(util.load_file(util.target_path( + resolvconf = ResolvConf(util.load_file(subp.target_path( target, self.resolv_conf_fn))) resolvconf.parse() except IOError: util.logexc(LOG, "Failed to parse %s, use new empty file", - util.target_path(target, self.resolv_conf_fn)) + subp.target_path(target, self.resolv_conf_fn)) resolvconf = ResolvConf('') resolvconf.parse() @@ -134,7 +135,7 @@ class BSDRenderer(renderer.Renderer): except ValueError: util.logexc(LOG, "Failed to add search domain %s", domain) util.write_file( - util.target_path(target, self.resolv_conf_fn), + subp.target_path(target, self.resolv_conf_fn), str(resolvconf), 0o644) def render_network_state(self, network_state, templates=None, target=None): diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 19d0199c..d03baeab 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -17,6 +17,7 @@ from cloudinit.net import ( has_url_connectivity) from cloudinit.net.network_state import mask_and_ipv4_to_bcast_addr as bcip from cloudinit import temp_utils +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -150,7 +151,7 @@ def maybe_perform_dhcp_discovery(nic=None): LOG.debug( 'Skip dhcp_discovery: nic %s not found in get_devicelist.', nic) return [] - dhclient_path = util.which('dhclient') + dhclient_path = subp.which('dhclient') if not dhclient_path: LOG.debug('Skip dhclient configuration: No dhclient command found.') return [] @@ -219,10 +220,10 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir): # Generally dhclient relies on dhclient-script PREINIT action to bring the # link up before attempting discovery. Since we are using -sf /bin/true, # we need to do that "link up" ourselves first. - util.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True) + subp.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True) cmd = [sandbox_dhclient_cmd, '-1', '-v', '-lf', lease_file, '-pf', pid_file, interface, '-sf', '/bin/true'] - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) # Wait for pid file and lease file to appear, and for the process # named by the pid file to daemonize (have pid 1 as its parent). If we diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 2f714563..b4c69457 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -11,6 +11,7 @@ from . import renderer from .network_state import subnet_is_ipv6 from cloudinit import log as logging +from cloudinit import subp from cloudinit import util @@ -511,13 +512,13 @@ class Renderer(renderer.Renderer): return '\n\n'.join(['\n'.join(s) for s in sections]) + "\n" def render_network_state(self, network_state, templates=None, target=None): - fpeni = util.target_path(target, self.eni_path) + fpeni = subp.target_path(target, self.eni_path) util.ensure_dir(os.path.dirname(fpeni)) header = self.eni_header if self.eni_header else "" util.write_file(fpeni, header + self._render_interfaces(network_state)) if self.netrules_path: - netrules = util.target_path(target, self.netrules_path) + netrules = subp.target_path(target, self.netrules_path) util.ensure_dir(os.path.dirname(netrules)) util.write_file(netrules, self._render_persistent_net(network_state)) @@ -544,9 +545,9 @@ def available(target=None): expected = ['ifquery', 'ifup', 'ifdown'] search = ['/sbin', '/usr/sbin'] for p in expected: - if not util.which(p, search=search, target=target): + if not subp.which(p, search=search, target=target): return False - eni = util.target_path(target, 'etc/network/interfaces') + eni = subp.target_path(target, 'etc/network/interfaces') if not os.path.isfile(eni): return False diff --git a/cloudinit/net/freebsd.py b/cloudinit/net/freebsd.py index 60f05bb2..0285dfec 100644 --- a/cloudinit/net/freebsd.py +++ b/cloudinit/net/freebsd.py @@ -2,6 +2,7 @@ from cloudinit import log as logging import cloudinit.net.bsd +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -30,17 +31,17 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): LOG.debug("freebsd generate postcmd disabled") return - util.subp(['service', 'netif', 'restart'], capture=True) + subp.subp(['service', 'netif', 'restart'], capture=True) # On FreeBSD 10, the restart of routing and dhclient is likely to fail # because # - routing: it cannot remove the loopback route, but it will still set # up the default route as expected. # - dhclient: it cannot stop the dhclient started by the netif service. # In both case, the situation is ok, and we can proceed. - util.subp(['service', 'routing', 'restart'], capture=True, rcs=[0, 1]) + subp.subp(['service', 'routing', 'restart'], capture=True, rcs=[0, 1]) for dhcp_interface in self.dhcp_interfaces(): - util.subp(['service', 'dhclient', 'restart', dhcp_interface], + subp.subp(['service', 'dhclient', 'restart', dhcp_interface], rcs=[0, 1], capture=True) diff --git a/cloudinit/net/netbsd.py b/cloudinit/net/netbsd.py index 9cc8ef31..30437b5f 100644 --- a/cloudinit/net/netbsd.py +++ b/cloudinit/net/netbsd.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import log as logging +from cloudinit import subp from cloudinit import util import cloudinit.net.bsd @@ -29,9 +30,9 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): LOG.debug("netbsd generate postcmd disabled") return - util.subp(['service', 'network', 'restart'], capture=True) + subp.subp(['service', 'network', 'restart'], capture=True) if self.dhcp_interfaces(): - util.subp(['service', 'dhcpcd', 'restart'], capture=True) + subp.subp(['service', 'dhcpcd', 'restart'], capture=True) def set_route(self, network, netmask, gateway): if network == '0.0.0.0': diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 89855270..53347c83 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -8,6 +8,7 @@ from .network_state import subnet_is_ipv6, NET_CONFIG_TO_V2, IPV6_DYNAMIC_TYPES from cloudinit import log as logging from cloudinit import util +from cloudinit import subp from cloudinit import safeyaml from cloudinit.net import SYS_CLASS_NET, get_devicelist @@ -164,14 +165,14 @@ def _extract_bond_slaves_by_name(interfaces, entry, bond_master): def _clean_default(target=None): # clean out any known default files and derived files in target # LP: #1675576 - tpath = util.target_path(target, "etc/netplan/00-snapd-config.yaml") + tpath = subp.target_path(target, "etc/netplan/00-snapd-config.yaml") if not os.path.isfile(tpath): return content = util.load_file(tpath, decode=False) if content != KNOWN_SNAPD_CONFIG: return - derived = [util.target_path(target, f) for f in ( + derived = [subp.target_path(target, f) for f in ( 'run/systemd/network/10-netplan-all-en.network', 'run/systemd/network/10-netplan-all-eth.network', 'run/systemd/generator/netplan.stamp')] @@ -203,10 +204,10 @@ class Renderer(renderer.Renderer): def features(self): if self._features is None: try: - info_blob, _err = util.subp(self.NETPLAN_INFO, capture=True) + info_blob, _err = subp.subp(self.NETPLAN_INFO, capture=True) info = util.load_yaml(info_blob) self._features = info['netplan.io']['features'] - except util.ProcessExecutionError: + except subp.ProcessExecutionError: # if the info subcommand is not present then we don't have any # new features pass @@ -218,7 +219,7 @@ class Renderer(renderer.Renderer): # check network state for version # if v2, then extract network_state.config # else render_v2_from_state - fpnplan = os.path.join(util.target_path(target), self.netplan_path) + fpnplan = os.path.join(subp.target_path(target), self.netplan_path) util.ensure_dir(os.path.dirname(fpnplan)) header = self.netplan_header if self.netplan_header else "" @@ -239,7 +240,7 @@ class Renderer(renderer.Renderer): if not run: LOG.debug("netplan generate postcmd disabled") return - util.subp(self.NETPLAN_GENERATE, capture=True) + subp.subp(self.NETPLAN_GENERATE, capture=True) def _net_setup_link(self, run=False): """To ensure device link properties are applied, we poke @@ -253,7 +254,7 @@ class Renderer(renderer.Renderer): for cmd in [setup_lnk + [SYS_CLASS_NET + iface] for iface in get_devicelist() if os.path.islink(SYS_CLASS_NET + iface)]: - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def _render_content(self, network_state): @@ -406,7 +407,7 @@ def available(target=None): expected = ['netplan'] search = ['/usr/sbin', '/sbin'] for p in expected: - if not util.which(p, search=search, target=target): + if not subp.which(p, search=search, target=target): return False return True diff --git a/cloudinit/net/openbsd.py b/cloudinit/net/openbsd.py index b9897e90..489ea48b 100644 --- a/cloudinit/net/openbsd.py +++ b/cloudinit/net/openbsd.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import log as logging +from cloudinit import subp from cloudinit import util import cloudinit.net.bsd @@ -12,7 +13,7 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): def write_config(self): for device_name, v in self.interface_configurations.items(): if_file = 'etc/hostname.{}'.format(device_name) - fn = util.target_path(self.target, if_file) + fn = subp.target_path(self.target, if_file) if device_name in self.dhcp_interfaces(): content = 'dhcp\n' elif isinstance(v, dict): @@ -30,12 +31,12 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): if not self._postcmds: LOG.debug("openbsd generate postcmd disabled") return - util.subp(['sh', '/etc/netstart'], capture=True) + subp.subp(['sh', '/etc/netstart'], capture=True) def set_route(self, network, netmask, gateway): if network == '0.0.0.0': if_file = 'etc/mygate' - fn = util.target_path(self.target, if_file) + fn = subp.target_path(self.target, if_file) content = gateway + '\n' util.write_file(fn, content) diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 0a387377..f36c300f 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -9,6 +9,7 @@ from configobj import ConfigObj from cloudinit import log as logging from cloudinit import util +from cloudinit import subp from cloudinit.distros.parsers import networkmanager_conf from cloudinit.distros.parsers import resolv_conf @@ -858,19 +859,19 @@ class Renderer(renderer.Renderer): if not templates: templates = self.templates file_mode = 0o644 - base_sysconf_dir = util.target_path(target, self.sysconf_dir) + base_sysconf_dir = subp.target_path(target, self.sysconf_dir) for path, data in self._render_sysconfig(base_sysconf_dir, network_state, self.flavor, templates=templates).items(): util.write_file(path, data, file_mode) if self.dns_path: - dns_path = util.target_path(target, self.dns_path) + dns_path = subp.target_path(target, self.dns_path) resolv_content = self._render_dns(network_state, existing_dns_path=dns_path) if resolv_content: util.write_file(dns_path, resolv_content, file_mode) if self.networkmanager_conf_path: - nm_conf_path = util.target_path(target, + nm_conf_path = subp.target_path(target, self.networkmanager_conf_path) nm_conf_content = self._render_networkmanager_conf(network_state, templates) @@ -878,12 +879,12 @@ class Renderer(renderer.Renderer): util.write_file(nm_conf_path, nm_conf_content, file_mode) if self.netrules_path: netrules_content = self._render_persistent_net(network_state) - netrules_path = util.target_path(target, self.netrules_path) + netrules_path = subp.target_path(target, self.netrules_path) util.write_file(netrules_path, netrules_content, file_mode) if available_nm(target=target): - enable_ifcfg_rh(util.target_path(target, path=NM_CFG_FILE)) + enable_ifcfg_rh(subp.target_path(target, path=NM_CFG_FILE)) - sysconfig_path = util.target_path(target, templates.get('control')) + sysconfig_path = subp.target_path(target, templates.get('control')) # Distros configuring /etc/sysconfig/network as a file e.g. Centos if sysconfig_path.endswith('network'): util.ensure_dir(os.path.dirname(sysconfig_path)) @@ -906,20 +907,20 @@ def available_sysconfig(target=None): expected = ['ifup', 'ifdown'] search = ['/sbin', '/usr/sbin'] for p in expected: - if not util.which(p, search=search, target=target): + if not subp.which(p, search=search, target=target): return False expected_paths = [ 'etc/sysconfig/network-scripts/network-functions', 'etc/sysconfig/config'] for p in expected_paths: - if os.path.isfile(util.target_path(target, p)): + if os.path.isfile(subp.target_path(target, p)): return True return False def available_nm(target=None): - if not os.path.isfile(util.target_path(target, path=NM_CFG_FILE)): + if not os.path.isfile(subp.target_path(target, path=NM_CFG_FILE)): return False return True diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index 7768da7c..d4881592 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -266,7 +266,7 @@ class TestDHCPDiscoveryClean(CiTestCase): 'Skip dhcp_discovery: nic idontexist not found in get_devicelist.', self.logs.getvalue()) - @mock.patch('cloudinit.net.dhcp.util.which') + @mock.patch('cloudinit.net.dhcp.subp.which') @mock.patch('cloudinit.net.dhcp.find_fallback_nic') def test_absent_dhclient_command(self, m_fallback, m_which): """When dhclient doesn't exist in the OS, log the issue and no-op.""" @@ -279,7 +279,7 @@ class TestDHCPDiscoveryClean(CiTestCase): @mock.patch('cloudinit.temp_utils.os.getuid') @mock.patch('cloudinit.net.dhcp.dhcp_discovery') - @mock.patch('cloudinit.net.dhcp.util.which') + @mock.patch('cloudinit.net.dhcp.subp.which') @mock.patch('cloudinit.net.dhcp.find_fallback_nic') def test_dhclient_run_with_tmpdir(self, m_fback, m_which, m_dhcp, m_uid): """maybe_perform_dhcp_discovery passes tmpdir to dhcp_discovery.""" @@ -302,7 +302,7 @@ class TestDHCPDiscoveryClean(CiTestCase): @mock.patch('time.sleep', mock.MagicMock()) @mock.patch('cloudinit.net.dhcp.os.kill') - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') def test_dhcp_discovery_run_in_sandbox_warns_invalid_pid(self, m_subp, m_kill): """dhcp_discovery logs a warning when pidfile contains invalid content. @@ -337,7 +337,7 @@ class TestDHCPDiscoveryClean(CiTestCase): @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid') @mock.patch('cloudinit.net.dhcp.os.kill') @mock.patch('cloudinit.net.dhcp.util.wait_for_files') - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') def test_dhcp_discovery_run_in_sandbox_waits_on_lease_and_pid(self, m_subp, m_wait, @@ -364,7 +364,7 @@ class TestDHCPDiscoveryClean(CiTestCase): @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid') @mock.patch('cloudinit.net.dhcp.os.kill') - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') def test_dhcp_discovery_run_in_sandbox(self, m_subp, m_kill, m_getppid): """dhcp_discovery brings up the interface and runs dhclient. @@ -529,7 +529,7 @@ class TestEphemeralDhcpNoNetworkSetup(HttprettyTestCase): # Ensure that no teardown happens: m_dhcp.assert_not_called() - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') def test_ephemeral_dhcp_setup_network_if_url_connectivity( self, m_dhcp, m_subp): diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index 835ed807..1c0a16a8 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -14,7 +14,8 @@ import requests import cloudinit.net as net from cloudinit import safeyaml as yaml from cloudinit.tests.helpers import CiTestCase, HttprettyTestCase -from cloudinit.util import ProcessExecutionError, ensure_file, write_file +from cloudinit.subp import ProcessExecutionError +from cloudinit.util import ensure_file, write_file class TestSysDevPath(CiTestCase): @@ -541,7 +542,7 @@ class TestInterfaceHasOwnMAC(CiTestCase): net.interface_has_own_mac('eth1', strict=True) -@mock.patch('cloudinit.net.util.subp') +@mock.patch('cloudinit.net.subp.subp') class TestEphemeralIPV4Network(CiTestCase): with_logs = True diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 1001f149..628e2908 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -13,6 +13,7 @@ import re from cloudinit import log as logging from cloudinit.net.network_state import net_prefix_to_ipv4_mask +from cloudinit import subp from cloudinit import util from cloudinit.simpletable import SimpleTable @@ -197,15 +198,15 @@ def _netdev_info_ifconfig(ifconfig_data): def netdev_info(empty=""): devs = {} if util.is_NetBSD(): - (ifcfg_out, _err) = util.subp(["ifconfig", "-a"], rcs=[0, 1]) + (ifcfg_out, _err) = subp.subp(["ifconfig", "-a"], rcs=[0, 1]) devs = _netdev_info_ifconfig_netbsd(ifcfg_out) - elif util.which('ip'): + elif subp.which('ip'): # Try iproute first of all - (ipaddr_out, _err) = util.subp(["ip", "addr", "show"]) + (ipaddr_out, _err) = subp.subp(["ip", "addr", "show"]) devs = _netdev_info_iproute(ipaddr_out) - elif util.which('ifconfig'): + elif subp.which('ifconfig'): # Fall back to net-tools if iproute2 is not present - (ifcfg_out, _err) = util.subp(["ifconfig", "-a"], rcs=[0, 1]) + (ifcfg_out, _err) = subp.subp(["ifconfig", "-a"], rcs=[0, 1]) devs = _netdev_info_ifconfig(ifcfg_out) else: LOG.warning( @@ -285,10 +286,10 @@ def _netdev_route_info_iproute(iproute_data): entry['flags'] = ''.join(flags) routes['ipv4'].append(entry) try: - (iproute_data6, _err6) = util.subp( + (iproute_data6, _err6) = subp.subp( ["ip", "--oneline", "-6", "route", "list", "table", "all"], rcs=[0, 1]) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass else: entries6 = iproute_data6.splitlines() @@ -357,9 +358,9 @@ def _netdev_route_info_netstat(route_data): routes['ipv4'].append(entry) try: - (route_data6, _err6) = util.subp( + (route_data6, _err6) = subp.subp( ["netstat", "-A", "inet6", "--route", "--numeric"], rcs=[0, 1]) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass else: entries6 = route_data6.splitlines() @@ -393,13 +394,13 @@ def _netdev_route_info_netstat(route_data): def route_info(): routes = {} - if util.which('ip'): + if subp.which('ip'): # Try iproute first of all - (iproute_out, _err) = util.subp(["ip", "-o", "route", "list"]) + (iproute_out, _err) = subp.subp(["ip", "-o", "route", "list"]) routes = _netdev_route_info_iproute(iproute_out) - elif util.which('netstat'): + elif subp.which('netstat'): # Fall back to net-tools if iproute2 is not present - (route_out, _err) = util.subp( + (route_out, _err) = subp.subp( ["netstat", "--route", "--numeric", "--extend"], rcs=[0, 1]) routes = _netdev_route_info_netstat(route_out) else: diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index 5270fda8..ac3ecc3d 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -18,9 +18,9 @@ import os.path from cloudinit import log as logging from cloudinit import sources +from cloudinit import subp from cloudinit import util -from cloudinit.util import ProcessExecutionError LOG = logging.getLogger(__name__) @@ -192,7 +192,7 @@ class DataSourceAltCloud(sources.DataSource): # modprobe floppy try: modprobe_floppy() - except ProcessExecutionError as e: + except subp.ProcessExecutionError as e: util.logexc(LOG, 'Failed modprobe: %s', e) return False @@ -201,7 +201,7 @@ class DataSourceAltCloud(sources.DataSource): # udevadm settle for floppy device try: util.udevadm_settle(exists=floppy_dev, timeout=5) - except (ProcessExecutionError, OSError) as e: + except (subp.ProcessExecutionError, OSError) as e: util.logexc(LOG, 'Failed udevadm_settle: %s\n', e) return False @@ -261,7 +261,7 @@ class DataSourceAltCloud(sources.DataSource): def modprobe_floppy(): - out, _err = util.subp(CMD_PROBE_FLOPPY) + out, _err = subp.subp(CMD_PROBE_FLOPPY) LOG.debug('Command: %s\nOutput%s', ' '.join(CMD_PROBE_FLOPPY), out) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 01d9adf2..89312b9e 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -22,6 +22,7 @@ from cloudinit.event import EventType from cloudinit.net.dhcp import EphemeralDHCPv4 from cloudinit import sources from cloudinit.sources.helpers import netlink +from cloudinit import subp from cloudinit.url_helper import UrlError, readurl, retry_on_url_exc from cloudinit import util from cloudinit.reporting import events @@ -139,8 +140,8 @@ def find_dev_from_busdev(camcontrol_out, busdev): def execute_or_debug(cmd, fail_ret=None): try: - return util.subp(cmd)[0] - except util.ProcessExecutionError: + return subp.subp(cmd)[0] + except subp.ProcessExecutionError: LOG.debug("Failed to execute: %s", ' '.join(cmd)) return fail_ret @@ -252,11 +253,11 @@ DEF_PASSWD_REDACTION = 'REDACTED' def get_hostname(hostname_command='hostname'): if not isinstance(hostname_command, (list, tuple)): hostname_command = (hostname_command,) - return util.subp(hostname_command, capture=True)[0].strip() + return subp.subp(hostname_command, capture=True)[0].strip() def set_hostname(hostname, hostname_command='hostname'): - util.subp([hostname_command, hostname]) + subp.subp([hostname_command, hostname]) @azure_ds_telemetry_reporter @@ -343,7 +344,7 @@ class DataSourceAzure(sources.DataSource): try: invoke_agent(agent_cmd) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: # claim the datasource even if the command failed util.logexc(LOG, "agent command '%s' failed.", self.ds_cfg['agent_command']) @@ -982,7 +983,7 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname): if command == "builtin": if util.is_FreeBSD(): command = BOUNCE_COMMAND_FREEBSD - elif util.which('ifup'): + elif subp.which('ifup'): command = BOUNCE_COMMAND_IFUP else: LOG.debug( @@ -993,7 +994,7 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname): shell = not isinstance(command, (list, tuple)) # capture=False, see comments in bug 1202758 and bug 1206164. util.log_time(logfunc=LOG.debug, msg="publishing hostname", - get_uptime=True, func=util.subp, + get_uptime=True, func=subp.subp, kwargs={'args': command, 'shell': shell, 'capture': False, 'env': env}) return True @@ -1003,7 +1004,7 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname): def crtfile_to_pubkey(fname, data=None): pipeline = ('openssl x509 -noout -pubkey < "$0" |' 'ssh-keygen -i -m PKCS8 -f /dev/stdin') - (out, _err) = util.subp(['sh', '-c', pipeline, fname], + (out, _err) = subp.subp(['sh', '-c', pipeline, fname], capture=True, data=data) return out.rstrip() @@ -1015,7 +1016,7 @@ def pubkeys_from_crt_files(flist): for fname in flist: try: pubkeys.append(crtfile_to_pubkey(fname)) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: errors.append(fname) if errors: @@ -1057,7 +1058,7 @@ def invoke_agent(cmd): # this is a function itself to simplify patching it for test if cmd: LOG.debug("invoking agent: %s", cmd) - util.subp(cmd, shell=(not isinstance(cmd, list))) + subp.subp(cmd, shell=(not isinstance(cmd, list))) else: LOG.debug("not invoking agent") diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 2013bed7..54810439 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -22,6 +22,7 @@ from cloudinit import log as logging from cloudinit.net import dhcp from cloudinit import sources from cloudinit import url_helper as uhelp +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -46,7 +47,7 @@ class CloudStackPasswordServerClient(object): # The password server was in the past, a broken HTTP server, but is now # fixed. wget handles this seamlessly, so it's easier to shell out to # that rather than write our own handling code. - output, _ = util.subp([ + output, _ = subp.subp([ 'wget', '--quiet', '--tries', '3', '--timeout', '20', '--output-document', '-', '--header', 'DomU_Request: {0}'.format(domu_request), diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index ae31934b..62756cf7 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -10,6 +10,7 @@ import os from cloudinit import log as logging from cloudinit import sources +from cloudinit import subp from cloudinit import util from cloudinit.net import eni @@ -245,7 +246,7 @@ def find_candidate_devs(probe_optical=True, dslist=None): for device in OPTICAL_DEVICES: try: util.find_devs_with(path=device) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass by_fstype = [] diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py index e0c714e8..d2aa9a58 100644 --- a/cloudinit/sources/DataSourceIBMCloud.py +++ b/cloudinit/sources/DataSourceIBMCloud.py @@ -99,6 +99,7 @@ import os from cloudinit import log as logging from cloudinit import sources from cloudinit.sources.helpers import openstack +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -240,7 +241,7 @@ def get_ibm_platform(): fslabels = {} try: devs = util.blkid() - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: LOG.warning("Failed to run blkid: %s", e) return (None, None) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 41f999e3..4c5eee4f 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -16,6 +16,7 @@ from xml.dom import minidom from cloudinit import log as logging from cloudinit import sources +from cloudinit import subp from cloudinit import util from cloudinit.sources.helpers.vmware.imc.config \ import Config @@ -536,15 +537,15 @@ def transport_iso9660(require_iso=True): def transport_vmware_guestinfo(): rpctool = "vmware-rpctool" not_found = None - if not util.which(rpctool): + if not subp.which(rpctool): return not_found cmd = [rpctool, "info-get guestinfo.ovfEnv"] try: - out, _err = util.subp(cmd) + out, _err = subp.subp(cmd) if out: return out LOG.debug("cmd %s exited 0 with empty stdout: %s", cmd, out) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code != 1: LOG.warning("%s exited with code %d", rpctool, e.exit_code) LOG.debug(e) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index a08ab404..c7fdc2a5 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -21,6 +21,7 @@ import string from cloudinit import log as logging from cloudinit import net from cloudinit import sources +from cloudinit import subp from cloudinit import util @@ -334,7 +335,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None, cmd.extend(bash) - (output, _error) = util.subp(cmd, data=bcmd) + (output, _error) = subp.subp(cmd, data=bcmd) # exclude vars in bash that change on their own or that we used excluded = ( @@ -396,7 +397,7 @@ def read_context_disk_dir(source_dir, asuser=None): path = os.path.join(source_dir, 'context.sh') content = util.load_file(path) context = parse_shell_config(content, asuser=asuser) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: raise BrokenContextDiskDir("Error processing context.sh: %s" % (e)) except IOError as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) diff --git a/cloudinit/sources/DataSourceRbxCloud.py b/cloudinit/sources/DataSourceRbxCloud.py index 084cb7d5..38fb5421 100644 --- a/cloudinit/sources/DataSourceRbxCloud.py +++ b/cloudinit/sources/DataSourceRbxCloud.py @@ -15,6 +15,7 @@ import os.path from cloudinit import log as logging from cloudinit import sources +from cloudinit import subp from cloudinit import util from cloudinit.event import EventType @@ -43,11 +44,11 @@ def int2ip(addr): def _sub_arp(cmd): """ - Uses the prefered cloud-init subprocess def of util.subp + Uses the prefered cloud-init subprocess def of subp.subp and runs arping. Breaking this to a separate function for later use in mocking and unittests """ - return util.subp(['arping'] + cmd) + return subp.subp(['arping'] + cmd) def gratuitous_arp(items, distro): @@ -61,7 +62,7 @@ def gratuitous_arp(items, distro): source_param, item['source'], item['destination'] ]) - except util.ProcessExecutionError as error: + except subp.ProcessExecutionError as error: # warning, because the system is able to function properly # despite no success - some ARP table may be waiting for # expiration, but the system may continue diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index cf676504..843b3a2a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -33,6 +33,7 @@ import socket from cloudinit import log as logging from cloudinit import serial from cloudinit import sources +from cloudinit import subp from cloudinit import util from cloudinit.event import EventType @@ -696,9 +697,9 @@ def identify_file(content_f): cmd = ["file", "--brief", "--mime-type", content_f] f_type = None try: - (f_type, _err) = util.subp(cmd) + (f_type, _err) = subp.subp(cmd) LOG.debug("script %s mime type is %s", content_f, f_type) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: util.logexc( LOG, ("Failed to identify script type for %s" % content_f, e)) return None if f_type is None else f_type.strip() diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 82b6730c..7bace8ca 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -15,6 +15,7 @@ from cloudinit import temp_utils from contextlib import contextmanager from xml.etree import ElementTree +from cloudinit import subp from cloudinit import url_helper from cloudinit import util from cloudinit import version @@ -92,7 +93,7 @@ def get_boot_telemetry(): raise RuntimeError("Failed to determine kernel start timestamp") try: - out, _ = util.subp(['/bin/systemctl', + out, _ = subp.subp(['/bin/systemctl', 'show', '-p', 'UserspaceTimestampMonotonic'], capture=True) @@ -105,7 +106,7 @@ def get_boot_telemetry(): "UserspaceTimestampMonotonic from systemd") user_start = kernel_start + (float(tsm) / 1000000) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: raise RuntimeError("Failed to get UserspaceTimestampMonotonic: %s" % e) except ValueError as e: @@ -114,7 +115,7 @@ def get_boot_telemetry(): % e) try: - out, _ = util.subp(['/bin/systemctl', 'show', + out, _ = subp.subp(['/bin/systemctl', 'show', 'cloud-init-local', '-p', 'InactiveExitTimestampMonotonic'], capture=True) @@ -126,7 +127,7 @@ def get_boot_telemetry(): "InactiveExitTimestampMonotonic from systemd") cloudinit_activation = kernel_start + (float(tsm) / 1000000) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: raise RuntimeError("Failed to get InactiveExitTimestampMonotonic: %s" % e) except ValueError as e: @@ -284,7 +285,7 @@ class OpenSSLManager(object): LOG.debug('Certificate already generated.') return with cd(self.tmpdir): - util.subp([ + subp.subp([ 'openssl', 'req', '-x509', '-nodes', '-subj', '/CN=LinuxTransport', '-days', '32768', '-newkey', 'rsa:2048', '-keyout', self.certificate_names['private_key'], @@ -301,14 +302,14 @@ class OpenSSLManager(object): @azure_ds_telemetry_reporter def _run_x509_action(action, cert): cmd = ['openssl', 'x509', '-noout', action] - result, _ = util.subp(cmd, data=cert) + result, _ = subp.subp(cmd, data=cert) return result @azure_ds_telemetry_reporter def _get_ssh_key_from_cert(self, certificate): pub_key = self._run_x509_action('-pubkey', certificate) keygen_cmd = ['ssh-keygen', '-i', '-m', 'PKCS8', '-f', '/dev/stdin'] - ssh_key, _ = util.subp(keygen_cmd, data=pub_key) + ssh_key, _ = subp.subp(keygen_cmd, data=pub_key) return ssh_key @azure_ds_telemetry_reporter @@ -341,7 +342,7 @@ class OpenSSLManager(object): certificates_content.encode('utf-8'), ] with cd(self.tmpdir): - out, _ = util.subp( + out, _ = subp.subp( 'openssl cms -decrypt -in /dev/stdin -inkey' ' {private_key} -recip {certificate} | openssl pkcs12 -nodes' ' -password pass:'.format(**self.certificate_names), diff --git a/cloudinit/sources/helpers/digitalocean.py b/cloudinit/sources/helpers/digitalocean.py index 0e7cccac..f5bbe46a 100644 --- a/cloudinit/sources/helpers/digitalocean.py +++ b/cloudinit/sources/helpers/digitalocean.py @@ -8,6 +8,7 @@ import random from cloudinit import log as logging from cloudinit import net as cloudnet from cloudinit import url_helper +from cloudinit import subp from cloudinit import util NIC_MAP = {'public': 'eth0', 'private': 'eth1'} @@ -36,14 +37,14 @@ def assign_ipv4_link_local(nic=None): ip_addr_cmd = ['ip', 'addr', 'add', addr, 'dev', nic] ip_link_cmd = ['ip', 'link', 'set', 'dev', nic, 'up'] - if not util.which('ip'): + if not subp.which('ip'): raise RuntimeError("No 'ip' command available to configure ip4LL " "address") try: - util.subp(ip_addr_cmd) + subp.subp(ip_addr_cmd) LOG.debug("assigned ip4LL address '%s' to '%s'", addr, nic) - util.subp(ip_link_cmd) + subp.subp(ip_link_cmd) LOG.debug("brought device '%s' up", nic) except Exception: util.logexc(LOG, "ip4LL address assignment of '%s' to '%s' failed." @@ -74,7 +75,7 @@ def del_ipv4_link_local(nic=None): ip_addr_cmd = ['ip', 'addr', 'flush', 'dev', nic] try: - util.subp(ip_addr_cmd) + subp.subp(ip_addr_cmd) LOG.debug("removed ip4LL addresses from %s", nic) except Exception as e: diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index a4373f24..c538720a 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -16,6 +16,7 @@ from cloudinit import ec2_utils from cloudinit import log as logging from cloudinit import net from cloudinit import sources +from cloudinit import subp from cloudinit import url_helper from cloudinit import util from cloudinit.sources import BrokenMetadata @@ -110,7 +111,7 @@ class SourceMixin(object): dev_entries = util.find_devs_with(criteria) if dev_entries: device = dev_entries[0] - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass return device diff --git a/cloudinit/sources/helpers/vmware/imc/config_custom_script.py b/cloudinit/sources/helpers/vmware/imc/config_custom_script.py index 9f14770e..2ab22de9 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_custom_script.py +++ b/cloudinit/sources/helpers/vmware/imc/config_custom_script.py @@ -9,6 +9,7 @@ import logging import os import stat +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -61,7 +62,7 @@ class PreCustomScript(RunCustomScript): """Executing custom script with precustomization argument.""" LOG.debug("Executing pre-customization script") self.prepare_script() - util.subp([CustomScriptConstant.CUSTOM_SCRIPT, "precustomization"]) + subp.subp([CustomScriptConstant.CUSTOM_SCRIPT, "precustomization"]) class PostCustomScript(RunCustomScript): diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 77cbf3b6..3745a262 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -10,6 +10,7 @@ import os import re from cloudinit.net.network_state import mask_to_net_prefix +from cloudinit import subp from cloudinit import util logger = logging.getLogger(__name__) @@ -73,7 +74,7 @@ class NicConfigurator(object): The mac address(es) are in the lower case """ cmd = ['ip', 'addr', 'show'] - output, _err = util.subp(cmd) + output, _err = subp.subp(cmd) sections = re.split(r'\n\d+: ', '\n' + output)[1:] macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))' @@ -248,8 +249,8 @@ class NicConfigurator(object): logger.info('Clearing DHCP leases') # Ignore the return code 1. - util.subp(["pkill", "dhclient"], rcs=[0, 1]) - util.subp(["rm", "-f", "/var/lib/dhcp/*"]) + subp.subp(["pkill", "dhclient"], rcs=[0, 1]) + subp.subp(["rm", "-f", "/var/lib/dhcp/*"]) def configure(self, osfamily=None): """ diff --git a/cloudinit/sources/helpers/vmware/imc/config_passwd.py b/cloudinit/sources/helpers/vmware/imc/config_passwd.py index 8c91fa41..d16a7690 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_passwd.py +++ b/cloudinit/sources/helpers/vmware/imc/config_passwd.py @@ -9,6 +9,7 @@ import logging import os +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -56,10 +57,10 @@ class PasswordConfigurator(object): LOG.info('Expiring password.') for user in uidUserList: try: - util.subp(['passwd', '--expire', user]) - except util.ProcessExecutionError as e: + subp.subp(['passwd', '--expire', user]) + except subp.ProcessExecutionError as e: if os.path.exists('/usr/bin/chage'): - util.subp(['chage', '-d', '0', user]) + subp.subp(['chage', '-d', '0', user]) else: LOG.warning('Failed to expire password for %s with error: ' '%s', user, e) diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index c60a38d7..893b1365 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -10,7 +10,7 @@ import os import re import time -from cloudinit import util +from cloudinit import subp from .guestcust_event import GuestCustEventEnum from .guestcust_state import GuestCustStateEnum @@ -34,7 +34,7 @@ def send_rpc(rpc): try: logger.debug("Sending RPC command: %s", rpc) - (out, err) = util.subp(["vmware-rpctool", rpc], rcs=[0]) + (out, err) = subp.subp(["vmware-rpctool", rpc], rcs=[0]) # Remove the trailing newline in the output. if out: out = out.rstrip() @@ -128,7 +128,7 @@ def get_tools_config(section, key, defaultVal): not installed. """ - if not util.which('vmware-toolbox-cmd'): + if not subp.which('vmware-toolbox-cmd'): logger.debug( 'vmware-toolbox-cmd not installed, returning default value') return defaultVal @@ -137,7 +137,7 @@ def get_tools_config(section, key, defaultVal): cmd = ['vmware-toolbox-cmd', 'config', 'get', section, key] try: - (outText, _) = util.subp(cmd) + (outText, _) = subp.subp(cmd) m = re.match(r'([^=]+)=(.*)', outText) if m: retValue = m.group(2).strip() @@ -147,7 +147,7 @@ def get_tools_config(section, key, defaultVal): logger.debug( "Tools config: [%s] %s is not found, return default value: %s", section, key, retValue) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: logger.error("Failed running %s[%s]", cmd, e.exit_code) logger.exception(e) diff --git a/cloudinit/subp.py b/cloudinit/subp.py index 0ad09306..f8400b1f 100644 --- a/cloudinit/subp.py +++ b/cloudinit/subp.py @@ -1,9 +1,11 @@ # This file is part of cloud-init. See LICENSE file for license information. """Common utility functions for interacting with subprocess.""" -# TODO move subp shellify and runparts related functions out of util.py - import logging +import os +import subprocess + +from errno import ENOEXEC LOG = logging.getLogger(__name__) @@ -54,4 +56,299 @@ def prepend_base_command(base_command, commands): return fixed_commands +class ProcessExecutionError(IOError): + + MESSAGE_TMPL = ('%(description)s\n' + 'Command: %(cmd)s\n' + 'Exit code: %(exit_code)s\n' + 'Reason: %(reason)s\n' + 'Stdout: %(stdout)s\n' + 'Stderr: %(stderr)s') + empty_attr = '-' + + def __init__(self, stdout=None, stderr=None, + exit_code=None, cmd=None, + description=None, reason=None, + errno=None): + if not cmd: + self.cmd = self.empty_attr + else: + self.cmd = cmd + + if not description: + if not exit_code and errno == ENOEXEC: + self.description = 'Exec format error. Missing #! in script?' + else: + self.description = 'Unexpected error while running command.' + else: + self.description = description + + if not isinstance(exit_code, int): + self.exit_code = self.empty_attr + else: + self.exit_code = exit_code + + if not stderr: + if stderr is None: + self.stderr = self.empty_attr + else: + self.stderr = stderr + else: + self.stderr = self._indent_text(stderr) + + if not stdout: + if stdout is None: + self.stdout = self.empty_attr + else: + self.stdout = stdout + else: + self.stdout = self._indent_text(stdout) + + if reason: + self.reason = reason + else: + self.reason = self.empty_attr + + self.errno = errno + message = self.MESSAGE_TMPL % { + 'description': self._ensure_string(self.description), + 'cmd': self._ensure_string(self.cmd), + 'exit_code': self._ensure_string(self.exit_code), + 'stdout': self._ensure_string(self.stdout), + 'stderr': self._ensure_string(self.stderr), + 'reason': self._ensure_string(self.reason), + } + IOError.__init__(self, message) + + def _ensure_string(self, text): + """ + if data is bytes object, decode + """ + return text.decode() if isinstance(text, bytes) else text + + def _indent_text(self, text, indent_level=8): + """ + indent text on all but the first line, allowing for easy to read output + """ + cr = '\n' + indent = ' ' * indent_level + # if input is bytes, return bytes + if isinstance(text, bytes): + cr = cr.encode() + indent = indent.encode() + # remove any newlines at end of text first to prevent unneeded blank + # line in output + return text.rstrip(cr).replace(cr, cr + indent) + + +def subp(args, data=None, rcs=None, env=None, capture=True, + combine_capture=False, shell=False, + logstring=False, decode="replace", target=None, update_env=None, + status_cb=None): + """Run a subprocess. + + :param args: command to run in a list. [cmd, arg1, arg2...] + :param data: input to the command, made available on its stdin. + :param rcs: + a list of allowed return codes. If subprocess exits with a value not + in this list, a ProcessExecutionError will be raised. By default, + data is returned as a string. See 'decode' parameter. + :param env: a dictionary for the command's environment. + :param capture: + boolean indicating if output should be captured. If True, then stderr + and stdout will be returned. If False, they will not be redirected. + :param combine_capture: + boolean indicating if stderr should be redirected to stdout. When True, + interleaved stderr and stdout will be returned as the first element of + a tuple, the second will be empty string or bytes (per decode). + if combine_capture is True, then output is captured independent of + the value of capture. + :param shell: boolean indicating if this should be run with a shell. + :param logstring: + the command will be logged to DEBUG. If it contains info that should + not be logged, then logstring will be logged instead. + :param decode: + if False, no decoding will be done and returned stdout and stderr will + be bytes. Other allowed values are 'strict', 'ignore', and 'replace'. + These values are passed through to bytes().decode() as the 'errors' + parameter. There is no support for decoding to other than utf-8. + :param target: + not supported, kwarg present only to make function signature similar + to curtin's subp. + :param update_env: + update the enviornment for this command with this dictionary. + this will not affect the current processes os.environ. + :param status_cb: + call this fuction with a single string argument before starting + and after finishing. + + :return + if not capturing, return is (None, None) + if capturing, stdout and stderr are returned. + if decode: + entries in tuple will be python2 unicode or python3 string + if not decode: + entries in tuple will be python2 string or python3 bytes + """ + + # not supported in cloud-init (yet), for now kept in the call signature + # to ease maintaining code shared between cloud-init and curtin + if target is not None: + raise ValueError("target arg not supported by cloud-init") + + if rcs is None: + rcs = [0] + + devnull_fp = None + + if update_env: + if env is None: + env = os.environ + env = env.copy() + env.update(update_env) + + if target_path(target) != "/": + args = ['chroot', target] + list(args) + + if status_cb: + command = ' '.join(args) if isinstance(args, list) else args + status_cb('Begin run command: {command}\n'.format(command=command)) + if not logstring: + LOG.debug(("Running command %s with allowed return codes %s" + " (shell=%s, capture=%s)"), + args, rcs, shell, 'combine' if combine_capture else capture) + else: + LOG.debug(("Running hidden command to protect sensitive " + "input/output logstring: %s"), logstring) + + stdin = None + stdout = None + stderr = None + if capture: + stdout = subprocess.PIPE + stderr = subprocess.PIPE + if combine_capture: + stdout = subprocess.PIPE + stderr = subprocess.STDOUT + if data is None: + # using devnull assures any reads get null, rather + # than possibly waiting on input. + devnull_fp = open(os.devnull) + stdin = devnull_fp + else: + stdin = subprocess.PIPE + if not isinstance(data, bytes): + data = data.encode() + + # Popen converts entries in the arguments array from non-bytes to bytes. + # When locale is unset it may use ascii for that encoding which can + # cause UnicodeDecodeErrors. (LP: #1751051) + if isinstance(args, bytes): + bytes_args = args + elif isinstance(args, str): + bytes_args = args.encode("utf-8") + else: + bytes_args = [ + x if isinstance(x, bytes) else x.encode("utf-8") + for x in args] + try: + sp = subprocess.Popen(bytes_args, stdout=stdout, + stderr=stderr, stdin=stdin, + env=env, shell=shell) + (out, err) = sp.communicate(data) + except OSError as e: + if status_cb: + status_cb('ERROR: End run command: invalid command provided\n') + raise ProcessExecutionError( + cmd=args, reason=e, errno=e.errno, + stdout="-" if decode else b"-", + stderr="-" if decode else b"-") + finally: + if devnull_fp: + devnull_fp.close() + + # Just ensure blank instead of none. + if capture or combine_capture: + if not out: + out = b'' + if not err: + err = b'' + if decode: + def ldecode(data, m='utf-8'): + if not isinstance(data, bytes): + return data + return data.decode(m, decode) + + out = ldecode(out) + err = ldecode(err) + + rc = sp.returncode + if rc not in rcs: + if status_cb: + status_cb( + 'ERROR: End run command: exit({code})\n'.format(code=rc)) + raise ProcessExecutionError(stdout=out, stderr=err, + exit_code=rc, + cmd=args) + if status_cb: + status_cb('End run command: exit({code})\n'.format(code=rc)) + return (out, err) + + +def target_path(target, path=None): + # return 'path' inside target, accepting target as None + if target in (None, ""): + target = "/" + elif not isinstance(target, str): + raise ValueError("Unexpected input for target: %s" % target) + else: + target = os.path.abspath(target) + # abspath("//") returns "//" specifically for 2 slashes. + if target.startswith("//"): + target = target[1:] + + if not path: + return target + + # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /. + while len(path) and path[0] == "/": + path = path[1:] + + return os.path.join(target, path) + + +def which(program, search=None, target=None): + target = target_path(target) + + if os.path.sep in program: + # if program had a '/' in it, then do not search PATH + # 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls + # so effectively we set cwd to / (or target) + if is_exe(target_path(target, program)): + return program + + if search is None: + paths = [p.strip('"') for p in + os.environ.get("PATH", "").split(os.pathsep)] + if target == "/": + search = paths + else: + search = [p for p in paths if p.startswith("/")] + + # normalize path input + search = [os.path.abspath(p) for p in search] + + for path in search: + ppath = os.path.sep.join((path, program)) + if is_exe(target_path(target, ppath)): + return ppath + + return None + + +def is_exe(fpath): + # return boolean indicating if fpath exists and is executable. + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + # vi: ts=4 expandtab diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index f3ab7e8c..58f63b69 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -23,9 +23,10 @@ from cloudinit import distros from cloudinit import helpers as ch from cloudinit.sources import DataSourceNone from cloudinit.templater import JINJA_AVAILABLE +from cloudinit import subp from cloudinit import util -_real_subp = util.subp +_real_subp = subp.subp # Used for skipping tests SkipTest = unittest.SkipTest @@ -134,9 +135,9 @@ class CiTestCase(TestCase): self.old_handlers = self.logger.handlers self.logger.handlers = [handler] if self.allowed_subp is True: - util.subp = _real_subp + subp.subp = _real_subp else: - util.subp = self._fake_subp + subp.subp = self._fake_subp def _fake_subp(self, *args, **kwargs): if 'args' in kwargs: @@ -171,7 +172,7 @@ class CiTestCase(TestCase): # Remove the handler we setup logging.getLogger().handlers = self.old_handlers logging.getLogger().level = None - util.subp = _real_subp + subp.subp = _real_subp super(CiTestCase, self).tearDown() def tmp_dir(self, dir=None, cleanup=True): @@ -280,13 +281,13 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): mock.patch.object(mod, f, trap_func)) # Handle subprocess calls - func = getattr(util, 'subp') + func = getattr(subp, 'subp') def nsubp(*_args, **_kwargs): return ('', '') self.patched_funcs.enter_context( - mock.patch.object(util, 'subp', nsubp)) + mock.patch.object(subp, 'subp', nsubp)) def null_func(*_args, **_kwargs): return None diff --git a/cloudinit/tests/test_conftest.py b/cloudinit/tests/test_conftest.py index 773ef8fe..a6537248 100644 --- a/cloudinit/tests/test_conftest.py +++ b/cloudinit/tests/test_conftest.py @@ -1,6 +1,6 @@ import pytest -from cloudinit import util +from cloudinit import subp from cloudinit.tests.helpers import CiTestCase @@ -9,17 +9,17 @@ class TestDisableSubpUsage: def test_using_subp_raises_assertion_error(self): with pytest.raises(AssertionError): - util.subp(["some", "args"]) + subp.subp(["some", "args"]) def test_typeerrors_on_incorrect_usage(self): with pytest.raises(TypeError): # We are intentionally passing no value for a parameter, so: # pylint: disable=no-value-for-parameter - util.subp() + subp.subp() @pytest.mark.parametrize('disable_subp_usage', [False], indirect=True) def test_subp_usage_can_be_reenabled(self): - util.subp(['whoami']) + subp.subp(['whoami']) @pytest.mark.parametrize( 'disable_subp_usage', [['whoami'], 'whoami'], indirect=True) @@ -27,18 +27,18 @@ class TestDisableSubpUsage: # The two parameters test each potential invocation with a single # argument with pytest.raises(AssertionError) as excinfo: - util.subp(["some", "args"]) + subp.subp(["some", "args"]) assert "allowed: whoami" in str(excinfo.value) - util.subp(['whoami']) + subp.subp(['whoami']) @pytest.mark.parametrize( 'disable_subp_usage', [['whoami', 'bash']], indirect=True) def test_subp_usage_can_be_conditionally_reenabled_for_multiple_cmds(self): with pytest.raises(AssertionError) as excinfo: - util.subp(["some", "args"]) + subp.subp(["some", "args"]) assert "allowed: whoami,bash" in str(excinfo.value) - util.subp(['bash', '-c', 'true']) - util.subp(['whoami']) + subp.subp(['bash', '-c', 'true']) + subp.subp(['whoami']) class TestDisableSubpUsageInTestSubclass(CiTestCase): @@ -46,16 +46,16 @@ class TestDisableSubpUsageInTestSubclass(CiTestCase): def test_using_subp_raises_exception(self): with pytest.raises(Exception): - util.subp(["some", "args"]) + subp.subp(["some", "args"]) def test_typeerrors_on_incorrect_usage(self): with pytest.raises(TypeError): - util.subp() + subp.subp() def test_subp_usage_can_be_reenabled(self): _old_allowed_subp = self.allow_subp self.allowed_subp = True try: - util.subp(['bash', '-c', 'true']) + subp.subp(['bash', '-c', 'true']) finally: self.allowed_subp = _old_allowed_subp diff --git a/cloudinit/tests/test_gpg.py b/cloudinit/tests/test_gpg.py index 8dd57137..f96f5372 100644 --- a/cloudinit/tests/test_gpg.py +++ b/cloudinit/tests/test_gpg.py @@ -4,19 +4,19 @@ from unittest import mock from cloudinit import gpg -from cloudinit import util +from cloudinit import subp from cloudinit.tests.helpers import CiTestCase @mock.patch("cloudinit.gpg.time.sleep") -@mock.patch("cloudinit.gpg.util.subp") +@mock.patch("cloudinit.gpg.subp.subp") class TestReceiveKeys(CiTestCase): """Test the recv_key method.""" def test_retries_on_subp_exc(self, m_subp, m_sleep): """retry should be done on gpg receive keys failure.""" retries = (1, 2, 4) - my_exc = util.ProcessExecutionError( + my_exc = subp.ProcessExecutionError( stdout='', stderr='', exit_code=2, cmd=['mycmd']) m_subp.side_effect = (my_exc, my_exc, ('', '')) gpg.recv_key("ABCD", "keyserver.example.com", retries=retries) @@ -26,7 +26,7 @@ class TestReceiveKeys(CiTestCase): """If the final run fails, error should be raised.""" naplen = 1 keyid, keyserver = ("ABCD", "keyserver.example.com") - m_subp.side_effect = util.ProcessExecutionError( + m_subp.side_effect = subp.ProcessExecutionError( stdout='', stderr='', exit_code=2, cmd=['mycmd']) with self.assertRaises(ValueError) as rcm: gpg.recv_key(keyid, keyserver, retries=(naplen,)) @@ -36,7 +36,7 @@ class TestReceiveKeys(CiTestCase): def test_no_retries_on_none(self, m_subp, m_sleep): """retry should not be done if retries is None.""" - m_subp.side_effect = util.ProcessExecutionError( + m_subp.side_effect = subp.ProcessExecutionError( stdout='', stderr='', exit_code=2, cmd=['mycmd']) with self.assertRaises(ValueError): gpg.recv_key("ABCD", "keyserver.example.com", retries=None) diff --git a/cloudinit/tests/test_netinfo.py b/cloudinit/tests/test_netinfo.py index 1c8a791e..e44b16d8 100644 --- a/cloudinit/tests/test_netinfo.py +++ b/cloudinit/tests/test_netinfo.py @@ -27,8 +27,8 @@ class TestNetInfo(CiTestCase): maxDiff = None with_logs = True - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_old_nettools_pformat(self, m_subp, m_which): """netdev_pformat properly rendering old nettools info.""" m_subp.return_value = (SAMPLE_OLD_IFCONFIG_OUT, '') @@ -36,8 +36,8 @@ class TestNetInfo(CiTestCase): content = netdev_pformat() self.assertEqual(NETDEV_FORMATTED_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_new_nettools_pformat(self, m_subp, m_which): """netdev_pformat properly rendering netdev new nettools info.""" m_subp.return_value = (SAMPLE_NEW_IFCONFIG_OUT, '') @@ -45,8 +45,8 @@ class TestNetInfo(CiTestCase): content = netdev_pformat() self.assertEqual(NETDEV_FORMATTED_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_freebsd_nettools_pformat(self, m_subp, m_which): """netdev_pformat properly rendering netdev new nettools info.""" m_subp.return_value = (SAMPLE_FREEBSD_IFCONFIG_OUT, '') @@ -57,8 +57,8 @@ class TestNetInfo(CiTestCase): print() self.assertEqual(FREEBSD_NETDEV_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_iproute_pformat(self, m_subp, m_which): """netdev_pformat properly rendering ip route info.""" m_subp.return_value = (SAMPLE_IPADDRSHOW_OUT, '') @@ -72,8 +72,8 @@ class TestNetInfo(CiTestCase): '255.0.0.0 | . |', '255.0.0.0 | host |') self.assertEqual(new_output, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_warn_on_missing_commands(self, m_subp, m_which): """netdev_pformat warns when missing both ip and 'netstat'.""" m_which.return_value = None # Niether ip nor netstat found @@ -85,8 +85,8 @@ class TestNetInfo(CiTestCase): self.logs.getvalue()) m_subp.assert_not_called() - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_info_nettools_down(self, m_subp, m_which): """test netdev_info using nettools and down interfaces.""" m_subp.return_value = ( @@ -100,8 +100,8 @@ class TestNetInfo(CiTestCase): 'hwaddr': '.', 'up': True}}, netdev_info(".")) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_info_iproute_down(self, m_subp, m_which): """Test netdev_info with ip and down interfaces.""" m_subp.return_value = ( @@ -130,8 +130,8 @@ class TestNetInfo(CiTestCase): readResource("netinfo/netdev-formatted-output-down"), netdev_pformat()) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_route_nettools_pformat(self, m_subp, m_which): """route_pformat properly rendering nettools route info.""" @@ -147,8 +147,8 @@ class TestNetInfo(CiTestCase): content = route_pformat() self.assertEqual(ROUTE_FORMATTED_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_route_iproute_pformat(self, m_subp, m_which): """route_pformat properly rendering ip route info.""" @@ -165,8 +165,8 @@ class TestNetInfo(CiTestCase): content = route_pformat() self.assertEqual(ROUTE_FORMATTED_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_route_warn_on_missing_commands(self, m_subp, m_which): """route_pformat warns when missing both ip and 'netstat'.""" m_which.return_value = None # Niether ip nor netstat found diff --git a/cloudinit/tests/test_subp.py b/cloudinit/tests/test_subp.py index 448097d3..911c1f3d 100644 --- a/cloudinit/tests/test_subp.py +++ b/cloudinit/tests/test_subp.py @@ -2,10 +2,21 @@ """Tests for cloudinit.subp utility functions""" -from cloudinit import subp +import json +import os +import sys +import stat + +from unittest import mock + +from cloudinit import subp, util from cloudinit.tests.helpers import CiTestCase +BASH = subp.which('bash') +BOGUS_COMMAND = 'this-is-not-expected-to-be-a-program-name' + + class TestPrependBaseCommands(CiTestCase): with_logs = True @@ -58,4 +69,218 @@ class TestPrependBaseCommands(CiTestCase): self.assertEqual('', self.logs.getvalue()) self.assertEqual(expected, fixed_commands) + +class TestSubp(CiTestCase): + allowed_subp = [BASH, 'cat', CiTestCase.SUBP_SHELL_TRUE, + BOGUS_COMMAND, sys.executable] + + stdin2err = [BASH, '-c', 'cat >&2'] + stdin2out = ['cat'] + utf8_invalid = b'ab\xaadef' + utf8_valid = b'start \xc3\xa9 end' + utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7' + printenv = [BASH, '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--'] + + def printf_cmd(self, *args): + # bash's printf supports \xaa. So does /usr/bin/printf + # but by using bash, we remove dependency on another program. + return([BASH, '-c', 'printf "$@"', 'printf'] + list(args)) + + def test_subp_handles_bytestrings(self): + """subp can run a bytestring command if shell is True.""" + tmp_file = self.tmp_path('test.out') + cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file) + (out, _err) = subp.subp(cmd.encode('utf-8'), shell=True) + self.assertEqual(u'', out) + self.assertEqual(u'', _err) + self.assertEqual('HI MOM\n', util.load_file(tmp_file)) + + def test_subp_handles_strings(self): + """subp can run a string command if shell is True.""" + tmp_file = self.tmp_path('test.out') + cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file) + (out, _err) = subp.subp(cmd, shell=True) + self.assertEqual(u'', out) + self.assertEqual(u'', _err) + self.assertEqual('HI MOM\n', util.load_file(tmp_file)) + + def test_subp_handles_utf8(self): + # The given bytes contain utf-8 accented characters as seen in e.g. + # the "deja dup" package in Ubuntu. + cmd = self.printf_cmd(self.utf8_valid_2) + (out, _err) = subp.subp(cmd, capture=True) + self.assertEqual(out, self.utf8_valid_2.decode('utf-8')) + + def test_subp_respects_decode_false(self): + (out, err) = subp.subp(self.stdin2out, capture=True, decode=False, + data=self.utf8_valid) + self.assertTrue(isinstance(out, bytes)) + self.assertTrue(isinstance(err, bytes)) + self.assertEqual(out, self.utf8_valid) + + def test_subp_decode_ignore(self): + # this executes a string that writes invalid utf-8 to stdout + (out, _err) = subp.subp(self.printf_cmd('abc\\xaadef'), + capture=True, decode='ignore') + self.assertEqual(out, 'abcdef') + + def test_subp_decode_strict_valid_utf8(self): + (out, _err) = subp.subp(self.stdin2out, capture=True, + decode='strict', data=self.utf8_valid) + self.assertEqual(out, self.utf8_valid.decode('utf-8')) + + def test_subp_decode_invalid_utf8_replaces(self): + (out, _err) = subp.subp(self.stdin2out, capture=True, + data=self.utf8_invalid) + expected = self.utf8_invalid.decode('utf-8', 'replace') + self.assertEqual(out, expected) + + def test_subp_decode_strict_raises(self): + args = [] + kwargs = {'args': self.stdin2out, 'capture': True, + 'decode': 'strict', 'data': self.utf8_invalid} + self.assertRaises(UnicodeDecodeError, subp.subp, *args, **kwargs) + + def test_subp_capture_stderr(self): + data = b'hello world' + (out, err) = subp.subp(self.stdin2err, capture=True, + decode=False, data=data, + update_env={'LC_ALL': 'C'}) + self.assertEqual(err, data) + self.assertEqual(out, b'') + + def test_subp_reads_env(self): + with mock.patch.dict("os.environ", values={'FOO': 'BAR'}): + out, _err = subp.subp(self.printenv + ['FOO'], capture=True) + self.assertEqual('FOO=BAR', out.splitlines()[0]) + + def test_subp_env_and_update_env(self): + out, _err = subp.subp( + self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True, + env={'FOO': 'BAR'}, + update_env={'HOME': '/myhome', 'K2': 'V2'}) + self.assertEqual( + ['FOO=BAR', 'HOME=/myhome', 'K1=', 'K2=V2'], out.splitlines()) + + def test_subp_update_env(self): + extra = {'FOO': 'BAR', 'HOME': '/root', 'K1': 'V1'} + with mock.patch.dict("os.environ", values=extra): + out, _err = subp.subp( + self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True, + update_env={'HOME': '/myhome', 'K2': 'V2'}) + + self.assertEqual( + ['FOO=BAR', 'HOME=/myhome', 'K1=V1', 'K2=V2'], out.splitlines()) + + def test_subp_warn_missing_shebang(self): + """Warn on no #! in script""" + noshebang = self.tmp_path('noshebang') + util.write_file(noshebang, 'true\n') + + print("os is %s" % os) + os.chmod(noshebang, os.stat(noshebang).st_mode | stat.S_IEXEC) + with self.allow_subp([noshebang]): + self.assertRaisesRegex(subp.ProcessExecutionError, + r'Missing #! in script\?', + subp.subp, (noshebang,)) + + def test_subp_combined_stderr_stdout(self): + """Providing combine_capture as True redirects stderr to stdout.""" + data = b'hello world' + (out, err) = subp.subp(self.stdin2err, capture=True, + combine_capture=True, decode=False, data=data) + self.assertEqual(b'', err) + self.assertEqual(data, out) + + def test_returns_none_if_no_capture(self): + (out, err) = subp.subp(self.stdin2out, data=b'', capture=False) + self.assertIsNone(err) + self.assertIsNone(out) + + def test_exception_has_out_err_are_bytes_if_decode_false(self): + """Raised exc should have stderr, stdout as bytes if no decode.""" + with self.assertRaises(subp.ProcessExecutionError) as cm: + subp.subp([BOGUS_COMMAND], decode=False) + self.assertTrue(isinstance(cm.exception.stdout, bytes)) + self.assertTrue(isinstance(cm.exception.stderr, bytes)) + + def test_exception_has_out_err_are_bytes_if_decode_true(self): + """Raised exc should have stderr, stdout as string if no decode.""" + with self.assertRaises(subp.ProcessExecutionError) as cm: + subp.subp([BOGUS_COMMAND], decode=True) + self.assertTrue(isinstance(cm.exception.stdout, str)) + self.assertTrue(isinstance(cm.exception.stderr, str)) + + def test_bunch_of_slashes_in_path(self): + self.assertEqual("/target/my/path/", + subp.target_path("/target/", "//my/path/")) + self.assertEqual("/target/my/path/", + subp.target_path("/target/", "///my/path/")) + + def test_c_lang_can_take_utf8_args(self): + """Independent of system LC_CTYPE, args can contain utf-8 strings. + + When python starts up, its default encoding gets set based on + the value of LC_CTYPE. If no system locale is set, the default + encoding for both python2 and python3 in some paths will end up + being ascii. + + Attempts to use setlocale or patching (or changing) os.environ + in the current environment seem to not be effective. + + This test starts up a python with LC_CTYPE set to C so that + the default encoding will be set to ascii. In such an environment + Popen(['command', 'non-ascii-arg']) would cause a UnicodeDecodeError. + """ + python_prog = '\n'.join([ + 'import json, sys', + 'from cloudinit.subp import subp', + 'data = sys.stdin.read()', + 'cmd = json.loads(data)', + 'subp(cmd, capture=False)', + '']) + cmd = [BASH, '-c', 'echo -n "$@"', '--', + self.utf8_valid.decode("utf-8")] + python_subp = [sys.executable, '-c', python_prog] + + out, _err = subp.subp( + python_subp, update_env={'LC_CTYPE': 'C'}, + data=json.dumps(cmd).encode("utf-8"), + decode=False) + self.assertEqual(self.utf8_valid, out) + + def test_bogus_command_logs_status_messages(self): + """status_cb gets status messages logs on bogus commands provided.""" + logs = [] + + def status_cb(log): + logs.append(log) + + with self.assertRaises(subp.ProcessExecutionError): + subp.subp([BOGUS_COMMAND], status_cb=status_cb) + + expected = [ + 'Begin run command: {cmd}\n'.format(cmd=BOGUS_COMMAND), + 'ERROR: End run command: invalid command provided\n'] + self.assertEqual(expected, logs) + + def test_command_logs_exit_codes_to_status_cb(self): + """status_cb gets status messages containing command exit code.""" + logs = [] + + def status_cb(log): + logs.append(log) + + with self.assertRaises(subp.ProcessExecutionError): + subp.subp([BASH, '-c', 'exit 2'], status_cb=status_cb) + subp.subp([BASH, '-c', 'exit 0'], status_cb=status_cb) + + expected = [ + 'Begin run command: %s -c exit 2\n' % BASH, + 'ERROR: End run command: exit(2)\n', + 'Begin run command: %s -c exit 0\n' % BASH, + 'End run command: exit(0)\n'] + self.assertEqual(expected, logs) + + # vi: ts=4 expandtab diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index bfccfe1e..0d9a8fd9 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -9,6 +9,7 @@ import platform import pytest import cloudinit.util as util +from cloudinit import subp from cloudinit.tests.helpers import CiTestCase, mock from textwrap import dedent @@ -332,7 +333,7 @@ class TestBlkid(CiTestCase): "PARTUUID": self.ids["id09"]}, }) - @mock.patch("cloudinit.util.subp") + @mock.patch("cloudinit.subp.subp") def test_functional_blkid(self, m_subp): m_subp.return_value = ( self.blkid_out.format(**self.ids), "") @@ -340,7 +341,7 @@ class TestBlkid(CiTestCase): m_subp.assert_called_with(["blkid", "-o", "full"], capture=True, decode="replace") - @mock.patch("cloudinit.util.subp") + @mock.patch("cloudinit.subp.subp") def test_blkid_no_cache_uses_no_cache(self, m_subp): """blkid should turn off cache if disable_cache is true.""" m_subp.return_value = ( @@ -351,7 +352,7 @@ class TestBlkid(CiTestCase): capture=True, decode="replace") -@mock.patch('cloudinit.util.subp') +@mock.patch('cloudinit.subp.subp') class TestUdevadmSettle(CiTestCase): def test_with_no_params(self, m_subp): """called with no parameters.""" @@ -396,8 +397,8 @@ class TestUdevadmSettle(CiTestCase): '--timeout=%s' % timeout]) def test_subp_exception_raises_to_caller(self, m_subp): - m_subp.side_effect = util.ProcessExecutionError("BOOM") - self.assertRaises(util.ProcessExecutionError, util.udevadm_settle) + m_subp.side_effect = subp.ProcessExecutionError("BOOM") + self.assertRaises(subp.ProcessExecutionError, util.udevadm_settle) @mock.patch('os.path.exists') diff --git a/cloudinit/util.py b/cloudinit/util.py index 985e7d20..445e3d4c 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -32,12 +32,13 @@ import subprocess import sys import time from base64 import b64decode, b64encode -from errno import ENOENT, ENOEXEC +from errno import ENOENT from functools import lru_cache from urllib import parse from cloudinit import importer from cloudinit import log as logging +from cloudinit import subp from cloudinit import ( mergers, safeyaml, @@ -74,8 +75,8 @@ def get_dpkg_architecture(target=None): N.B. This function is wrapped in functools.lru_cache, so repeated calls won't shell out every time. """ - out, _ = subp(['dpkg', '--print-architecture'], capture=True, - target=target) + out, _ = subp.subp(['dpkg', '--print-architecture'], capture=True, + target=target) return out.strip() @@ -86,7 +87,8 @@ def lsb_release(target=None): data = {} try: - out, _ = subp(['lsb_release', '--all'], capture=True, target=target) + out, _ = subp.subp(['lsb_release', '--all'], capture=True, + target=target) for line in out.splitlines(): fname, _, val = line.partition(":") if fname in fmap: @@ -96,35 +98,13 @@ def lsb_release(target=None): LOG.warning("Missing fields in lsb_release --all output: %s", ','.join(missing)) - except ProcessExecutionError as err: + except subp.ProcessExecutionError as err: LOG.warning("Unable to get lsb_release --all: %s", err) data = dict((v, "UNAVAILABLE") for v in fmap.values()) return data -def target_path(target, path=None): - # return 'path' inside target, accepting target as None - if target in (None, ""): - target = "/" - elif not isinstance(target, str): - raise ValueError("Unexpected input for target: %s" % target) - else: - target = os.path.abspath(target) - # abspath("//") returns "//" specifically for 2 slashes. - if target.startswith("//"): - target = target[1:] - - if not path: - return target - - # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /. - while len(path) and path[0] == "/": - path = path[1:] - - return os.path.join(target, path) - - def decode_binary(blob, encoding='utf-8'): # Converts a binary type into a text type using given encoding. if isinstance(blob, str): @@ -201,91 +181,6 @@ DMIDECODE_TO_DMI_SYS_MAPPING = { } -class ProcessExecutionError(IOError): - - MESSAGE_TMPL = ('%(description)s\n' - 'Command: %(cmd)s\n' - 'Exit code: %(exit_code)s\n' - 'Reason: %(reason)s\n' - 'Stdout: %(stdout)s\n' - 'Stderr: %(stderr)s') - empty_attr = '-' - - def __init__(self, stdout=None, stderr=None, - exit_code=None, cmd=None, - description=None, reason=None, - errno=None): - if not cmd: - self.cmd = self.empty_attr - else: - self.cmd = cmd - - if not description: - if not exit_code and errno == ENOEXEC: - self.description = 'Exec format error. Missing #! in script?' - else: - self.description = 'Unexpected error while running command.' - else: - self.description = description - - if not isinstance(exit_code, int): - self.exit_code = self.empty_attr - else: - self.exit_code = exit_code - - if not stderr: - if stderr is None: - self.stderr = self.empty_attr - else: - self.stderr = stderr - else: - self.stderr = self._indent_text(stderr) - - if not stdout: - if stdout is None: - self.stdout = self.empty_attr - else: - self.stdout = stdout - else: - self.stdout = self._indent_text(stdout) - - if reason: - self.reason = reason - else: - self.reason = self.empty_attr - - self.errno = errno - message = self.MESSAGE_TMPL % { - 'description': self._ensure_string(self.description), - 'cmd': self._ensure_string(self.cmd), - 'exit_code': self._ensure_string(self.exit_code), - 'stdout': self._ensure_string(self.stdout), - 'stderr': self._ensure_string(self.stderr), - 'reason': self._ensure_string(self.reason), - } - IOError.__init__(self, message) - - def _ensure_string(self, text): - """ - if data is bytes object, decode - """ - return text.decode() if isinstance(text, bytes) else text - - def _indent_text(self, text, indent_level=8): - """ - indent text on all but the first line, allowing for easy to read output - """ - cr = '\n' - indent = ' ' * indent_level - # if input is bytes, return bytes - if isinstance(text, bytes): - cr = cr.encode() - indent = indent.encode() - # remove any newlines at end of text first to prevent unneeded blank - # line in output - return text.rstrip(cr).replace(cr, cr + indent) - - class SeLinuxGuard(object): def __init__(self, path, recursive=False): # Late import since it might not always @@ -875,8 +770,8 @@ def runparts(dirp, skip_no_exist=True, exe_prefix=None): if os.path.isfile(exe_path) and os.access(exe_path, os.X_OK): attempted.append(exe_path) try: - subp(prefix + [exe_path], capture=False) - except ProcessExecutionError as e: + subp.subp(prefix + [exe_path], capture=False) + except subp.ProcessExecutionError as e: logexc(LOG, "Failed running %s [%s]", exe_path, e.exit_code) failed.append(e) @@ -1271,10 +1166,10 @@ def find_devs_with_netbsd(criteria=None, oformat='device', label = criteria.lstrip("LABEL=") if criteria.startswith("TYPE="): _type = criteria.lstrip("TYPE=") - out, _err = subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) + out, _err = subp.subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) for dev in out.split(): if label or _type: - mscdlabel_out, _ = subp(['mscdlabel', dev], rcs=[0, 1]) + mscdlabel_out, _ = subp.subp(['mscdlabel', dev], rcs=[0, 1]) if label and not ('label "%s"' % label) in mscdlabel_out: continue if _type == "iso9660" and "ISO filesystem" not in mscdlabel_out: @@ -1287,7 +1182,7 @@ def find_devs_with_netbsd(criteria=None, oformat='device', def find_devs_with_openbsd(criteria=None, oformat='device', tag=None, no_cache=False, path=None): - out, _err = subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) + out, _err = subp.subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) devlist = [] for entry in out.split(','): if not entry.endswith(':'): @@ -1353,8 +1248,8 @@ def find_devs_with(criteria=None, oformat='device', cmd = blk_id_cmd + options # See man blkid for why 2 is added try: - (out, _err) = subp(cmd, rcs=[0, 2]) - except ProcessExecutionError as e: + (out, _err) = subp.subp(cmd, rcs=[0, 2]) + except subp.ProcessExecutionError as e: if e.errno == ENOENT: # blkid not found... out = "" @@ -1389,7 +1284,7 @@ def blkid(devs=None, disable_cache=False): # we have to decode with 'replace' as shelx.split (called by # load_shell_content) can't take bytes. So this is potentially # lossy of non-utf-8 chars in blkid output. - out, _ = subp(cmd, capture=True, decode="replace") + out, _ = subp.subp(cmd, capture=True, decode="replace") ret = {} for line in out.splitlines(): dev, _, data = line.partition(":") @@ -1709,7 +1604,7 @@ def unmounter(umount): finally: if umount: umount_cmd = ["umount", umount] - subp(umount_cmd) + subp.subp(umount_cmd) def mounts(): @@ -1720,7 +1615,7 @@ def mounts(): mount_locs = load_file("/proc/mounts").splitlines() method = 'proc' else: - (mountoutput, _err) = subp("mount") + (mountoutput, _err) = subp.subp("mount") mount_locs = mountoutput.splitlines() method = 'mount' mountre = r'^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$' @@ -1804,7 +1699,7 @@ def mount_cb(device, callback, data=None, mtype=None, mountcmd.extend(['-t', mtype]) mountcmd.append(device) mountcmd.append(tmpd) - subp(mountcmd, update_env=update_env_for_mount) + subp.subp(mountcmd, update_env=update_env_for_mount) umount = tmpd # This forces it to be unmounted (when set) mountpoint = tmpd break @@ -1988,185 +1883,6 @@ def delete_dir_contents(dirname): del_file(node_fullpath) -def subp_blob_in_tempfile(blob, *args, **kwargs): - """Write blob to a tempfile, and call subp with args, kwargs. Then cleanup. - - 'basename' as a kwarg allows providing the basename for the file. - The 'args' argument to subp will be updated with the full path to the - filename as the first argument. - """ - basename = kwargs.pop('basename', "subp_blob") - - if len(args) == 0 and 'args' not in kwargs: - args = [tuple()] - - # Use tmpdir over tmpfile to avoid 'text file busy' on execute - with temp_utils.tempdir(needs_exe=True) as tmpd: - tmpf = os.path.join(tmpd, basename) - if 'args' in kwargs: - kwargs['args'] = [tmpf] + list(kwargs['args']) - else: - args = list(args) - args[0] = [tmpf] + args[0] - - write_file(tmpf, blob, mode=0o700) - return subp(*args, **kwargs) - - -def subp(args, data=None, rcs=None, env=None, capture=True, - combine_capture=False, shell=False, - logstring=False, decode="replace", target=None, update_env=None, - status_cb=None): - """Run a subprocess. - - :param args: command to run in a list. [cmd, arg1, arg2...] - :param data: input to the command, made available on its stdin. - :param rcs: - a list of allowed return codes. If subprocess exits with a value not - in this list, a ProcessExecutionError will be raised. By default, - data is returned as a string. See 'decode' parameter. - :param env: a dictionary for the command's environment. - :param capture: - boolean indicating if output should be captured. If True, then stderr - and stdout will be returned. If False, they will not be redirected. - :param combine_capture: - boolean indicating if stderr should be redirected to stdout. When True, - interleaved stderr and stdout will be returned as the first element of - a tuple, the second will be empty string or bytes (per decode). - if combine_capture is True, then output is captured independent of - the value of capture. - :param shell: boolean indicating if this should be run with a shell. - :param logstring: - the command will be logged to DEBUG. If it contains info that should - not be logged, then logstring will be logged instead. - :param decode: - if False, no decoding will be done and returned stdout and stderr will - be bytes. Other allowed values are 'strict', 'ignore', and 'replace'. - These values are passed through to bytes().decode() as the 'errors' - parameter. There is no support for decoding to other than utf-8. - :param target: - not supported, kwarg present only to make function signature similar - to curtin's subp. - :param update_env: - update the enviornment for this command with this dictionary. - this will not affect the current processes os.environ. - :param status_cb: - call this fuction with a single string argument before starting - and after finishing. - - :return - if not capturing, return is (None, None) - if capturing, stdout and stderr are returned. - if decode: - entries in tuple will be python2 unicode or python3 string - if not decode: - entries in tuple will be python2 string or python3 bytes - """ - - # not supported in cloud-init (yet), for now kept in the call signature - # to ease maintaining code shared between cloud-init and curtin - if target is not None: - raise ValueError("target arg not supported by cloud-init") - - if rcs is None: - rcs = [0] - - devnull_fp = None - - if update_env: - if env is None: - env = os.environ - env = env.copy() - env.update(update_env) - - if target_path(target) != "/": - args = ['chroot', target] + list(args) - - if status_cb: - command = ' '.join(args) if isinstance(args, list) else args - status_cb('Begin run command: {command}\n'.format(command=command)) - if not logstring: - LOG.debug(("Running command %s with allowed return codes %s" - " (shell=%s, capture=%s)"), - args, rcs, shell, 'combine' if combine_capture else capture) - else: - LOG.debug(("Running hidden command to protect sensitive " - "input/output logstring: %s"), logstring) - - stdin = None - stdout = None - stderr = None - if capture: - stdout = subprocess.PIPE - stderr = subprocess.PIPE - if combine_capture: - stdout = subprocess.PIPE - stderr = subprocess.STDOUT - if data is None: - # using devnull assures any reads get null, rather - # than possibly waiting on input. - devnull_fp = open(os.devnull) - stdin = devnull_fp - else: - stdin = subprocess.PIPE - if not isinstance(data, bytes): - data = data.encode() - - # Popen converts entries in the arguments array from non-bytes to bytes. - # When locale is unset it may use ascii for that encoding which can - # cause UnicodeDecodeErrors. (LP: #1751051) - if isinstance(args, bytes): - bytes_args = args - elif isinstance(args, str): - bytes_args = args.encode("utf-8") - else: - bytes_args = [ - x if isinstance(x, bytes) else x.encode("utf-8") - for x in args] - try: - sp = subprocess.Popen(bytes_args, stdout=stdout, - stderr=stderr, stdin=stdin, - env=env, shell=shell) - (out, err) = sp.communicate(data) - except OSError as e: - if status_cb: - status_cb('ERROR: End run command: invalid command provided\n') - raise ProcessExecutionError( - cmd=args, reason=e, errno=e.errno, - stdout="-" if decode else b"-", - stderr="-" if decode else b"-") - finally: - if devnull_fp: - devnull_fp.close() - - # Just ensure blank instead of none. - if capture or combine_capture: - if not out: - out = b'' - if not err: - err = b'' - if decode: - def ldecode(data, m='utf-8'): - if not isinstance(data, bytes): - return data - return data.decode(m, decode) - - out = ldecode(out) - err = ldecode(err) - - rc = sp.returncode - if rc not in rcs: - if status_cb: - status_cb( - 'ERROR: End run command: exit({code})\n'.format(code=rc)) - raise ProcessExecutionError(stdout=out, stderr=err, - exit_code=rc, - cmd=args) - if status_cb: - status_cb('End run command: exit({code})\n'.format(code=rc)) - return (out, err) - - def make_header(comment_char="#", base='created'): ci_ver = version.version_string() header = str(comment_char) @@ -2232,7 +1948,7 @@ def is_container(): try: # try to run a helper program. if it returns true/zero # then we're inside a container. otherwise, no - subp(helper) + subp.subp(helper) return True except (IOError, OSError): pass @@ -2438,7 +2154,7 @@ def find_freebsd_part(fs): return splitted[2] elif splitted[2] in ['label', 'gpt', 'ufs']: target_label = fs[5:] - (part, _err) = subp(['glabel', 'status', '-s']) + (part, _err) = subp.subp(['glabel', 'status', '-s']) for labels in part.split("\n"): items = labels.split() if len(items) > 0 and items[0] == target_label: @@ -2460,10 +2176,10 @@ def get_path_dev_freebsd(path, mnt_list): def get_mount_info_freebsd(path): - (result, err) = subp(['mount', '-p', path], rcs=[0, 1]) + (result, err) = subp.subp(['mount', '-p', path], rcs=[0, 1]) if len(err): # find a path if the input is not a mounting point - (mnt_list, err) = subp(['mount', '-p']) + (mnt_list, err) = subp.subp(['mount', '-p']) path_found = get_path_dev_freebsd(path, mnt_list) if (path_found is None): return None @@ -2479,8 +2195,8 @@ def get_device_info_from_zpool(zpool): LOG.debug('Cannot get zpool info, no /dev/zfs') return None try: - (zpoolstatus, err) = subp(['zpool', 'status', zpool]) - except ProcessExecutionError as err: + (zpoolstatus, err) = subp.subp(['zpool', 'status', zpool]) + except subp.ProcessExecutionError as err: LOG.warning("Unable to get zpool status of %s: %s", zpool, err) return None if len(err): @@ -2494,7 +2210,7 @@ def get_device_info_from_zpool(zpool): def parse_mount(path): - (mountoutput, _err) = subp(['mount']) + (mountoutput, _err) = subp.subp(['mount']) mount_locs = mountoutput.splitlines() # there are 2 types of mount outputs we have to parse therefore # the regex is a bit complex. to better understand this regex see: @@ -2567,40 +2283,6 @@ def get_mount_info(path, log=LOG, get_mnt_opts=False): return parse_mount(path) -def is_exe(fpath): - # return boolean indicating if fpath exists and is executable. - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - -def which(program, search=None, target=None): - target = target_path(target) - - if os.path.sep in program: - # if program had a '/' in it, then do not search PATH - # 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls - # so effectively we set cwd to / (or target) - if is_exe(target_path(target, program)): - return program - - if search is None: - paths = [p.strip('"') for p in - os.environ.get("PATH", "").split(os.pathsep)] - if target == "/": - search = paths - else: - search = [p for p in paths if p.startswith("/")] - - # normalize path input - search = [os.path.abspath(p) for p in search] - - for path in search: - ppath = os.path.sep.join((path, program)) - if is_exe(target_path(target, ppath)): - return ppath - - return None - - def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): if args is None: args = [] @@ -2764,7 +2446,7 @@ def _call_dmidecode(key, dmidecode_path): """ try: cmd = [dmidecode_path, "--string", key] - (result, _err) = subp(cmd) + (result, _err) = subp.subp(cmd) result = result.strip() LOG.debug("dmidecode returned '%s' for '%s'", result, key) if result.replace(".", "") == "": @@ -2818,7 +2500,8 @@ def read_dmi_data(key): LOG.debug("dmidata is not supported on %s", uname_arch) return None - dmidecode_path = which('dmidecode') + print("hi, now its: %s\n", subp) + dmidecode_path = subp.which('dmidecode') if dmidecode_path: return _call_dmidecode(key, dmidecode_path) @@ -2834,7 +2517,7 @@ def message_from_string(string): def get_installed_packages(target=None): - (out, _) = subp(['dpkg-query', '--list'], target=target, capture=True) + (out, _) = subp.subp(['dpkg-query', '--list'], target=target, capture=True) pkgs_inst = set() for line in out.splitlines(): @@ -2970,7 +2653,7 @@ def udevadm_settle(exists=None, timeout=None): if timeout: settle_cmd.extend(['--timeout=%s' % timeout]) - return subp(settle_cmd) + return subp.subp(settle_cmd) def get_proc_ppid(pid): |