From 1426a0d036a7f6707aac142a51f155f507d26739 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 13 Mar 2020 15:49:42 -0400 Subject: cc_resolv_conf: introduce tests and stabilise output across Python versions (#251) This is a follow-up to #144 which fixed the rendering behaviour. While writing the tests, CI failed due to dict iteration differences across Python versions, so this also sorts output so that we will produce the same output across Python versions. --- cloudinit/config/tests/test_resolv_conf.py | 86 ++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 cloudinit/config/tests/test_resolv_conf.py (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/tests/test_resolv_conf.py b/cloudinit/config/tests/test_resolv_conf.py new file mode 100644 index 00000000..f3d97b82 --- /dev/null +++ b/cloudinit/config/tests/test_resolv_conf.py @@ -0,0 +1,86 @@ +from unittest import mock + +import pytest + +from cloudinit.config.cc_resolv_conf import generate_resolv_conf + + +EXPECTED_HEADER = """\ +# Your system has been configured with 'manage-resolv-conf' set to true. +# As a result, cloud-init has written this file with configuration data +# that it has been provided. Cloud-init, by default, will write this file +# a single time (PER_ONCE). +#\n\n""" + + +class TestGenerateResolvConf: + @mock.patch("cloudinit.config.cc_resolv_conf.templater.render_to_file") + def test_default_target_fname_is_etc_resolvconf(self, m_render_to_file): + generate_resolv_conf("templates/resolv.conf.tmpl", mock.MagicMock()) + + assert [ + mock.call(mock.ANY, "/etc/resolv.conf", mock.ANY) + ] == m_render_to_file.call_args_list + + @mock.patch("cloudinit.config.cc_resolv_conf.templater.render_to_file") + def test_target_fname_is_used_if_passed(self, m_render_to_file): + generate_resolv_conf( + "templates/resolv.conf.tmpl", mock.MagicMock(), "/use/this/path" + ) + + assert [ + mock.call(mock.ANY, "/use/this/path", mock.ANY) + ] == m_render_to_file.call_args_list + + # Patch in templater so we can assert on the actual generated content + @mock.patch("cloudinit.templater.util.write_file") + # Parameterise with the value to be passed to generate_resolv_conf as the + # `params` parameter, and a list of the expected lines after the header as + # `extra_lines`. + @pytest.mark.parametrize( + "params,expected_extra_line", + [ + # No options + ({}, None), + # Just a true flag + ({"options": {"foo": True}}, "options foo"), + # Just a false flag + ({"options": {"foo": False}}, None), + # Just an option + ({"options": {"foo": "some_value"}}, "options foo:some_value"), + # A true flag and an option + ( + {"options": {"foo": "some_value", "bar": True}}, + "options bar foo:some_value", + ), + # Two options + ( + {"options": {"foo": "some_value", "bar": "other_value"}}, + "options bar:other_value foo:some_value", + ), + # Everything + ( + { + "options": { + "foo": "some_value", + "bar": "other_value", + "baz": False, + "spam": True, + } + }, + "options spam bar:other_value foo:some_value", + ), + ], + ) + def test_flags_and_options( + self, m_write_file, params, expected_extra_line + ): + generate_resolv_conf("templates/resolv.conf.tmpl", params) + + expected_content = EXPECTED_HEADER + if expected_extra_line is not None: + # If we have any extra lines, expect a trailing newline + expected_content += "\n".join([expected_extra_line, ""]) + assert [ + mock.call(mock.ANY, expected_content, mode=mock.ANY) + ] == m_write_file.call_args_list -- cgit v1.2.3 From ade47866aa2f81ab0f3baabbb1fc1e484abdb741 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 19 Mar 2020 12:01:15 -0400 Subject: cloudinit/tests: remove unneeded with_logs configuration (#263) These classes don't use `self.logs` anywhere in their body, so we can remove the `with_logs = True` setting from them. These instances were found using astpath[0], with the following invocation: astpath "//Name[@id='with_logs' and not(ancestor::ClassDef//Attribute[@attr='logs'])]" [0] https://github.com/hchasestevens/astpath --- cloudinit/cmd/tests/test_main.py | 2 -- cloudinit/config/tests/test_disable_ec2_metadata.py | 2 -- cloudinit/net/tests/test_init.py | 6 ------ cloudinit/sources/tests/test_oracle.py | 2 -- tests/unittests/test_datasource/test_azure.py | 2 -- tests/unittests/test_datasource/test_maas.py | 1 - tests/unittests/test_handler/test_handler_locale.py | 2 -- tests/unittests/test_handler/test_handler_puppet.py | 2 -- tests/unittests/test_util.py | 1 - 9 files changed, 20 deletions(-) (limited to 'cloudinit/config/tests') diff --git a/cloudinit/cmd/tests/test_main.py b/cloudinit/cmd/tests/test_main.py index 384fddc6..585b3b0e 100644 --- a/cloudinit/cmd/tests/test_main.py +++ b/cloudinit/cmd/tests/test_main.py @@ -18,8 +18,6 @@ myargs = namedtuple('MyArgs', 'debug files force local reporter subcommand') class TestMain(FilesystemMockingTestCase): - with_logs = True - def setUp(self): super(TestMain, self).setUp() self.new_root = self.tmp_dir() diff --git a/cloudinit/config/tests/test_disable_ec2_metadata.py b/cloudinit/config/tests/test_disable_ec2_metadata.py index 67646b03..823917c7 100644 --- a/cloudinit/config/tests/test_disable_ec2_metadata.py +++ b/cloudinit/config/tests/test_disable_ec2_metadata.py @@ -15,8 +15,6 @@ DISABLE_CFG = {'disable_ec2_metadata': 'true'} class TestEC2MetadataRoute(CiTestCase): - with_logs = True - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') def test_disable_ifconfig(self, m_subp, m_which): diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index 5081a337..a965699a 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -341,8 +341,6 @@ class TestGenerateFallbackConfig(CiTestCase): class TestNetFindFallBackNic(CiTestCase): - with_logs = True - def setUp(self): super(TestNetFindFallBackNic, self).setUp() sys_mock = mock.patch('cloudinit.net.get_sys_class_path') @@ -995,8 +993,6 @@ class TestExtractPhysdevs(CiTestCase): class TestWaitForPhysdevs(CiTestCase): - with_logs = True - def setUp(self): super(TestWaitForPhysdevs, self).setUp() self.add_patch('cloudinit.net.get_interfaces_by_mac', @@ -1071,8 +1067,6 @@ class TestWaitForPhysdevs(CiTestCase): class TestNetFailOver(CiTestCase): - with_logs = True - def setUp(self): super(TestNetFailOver, self).setUp() self.add_patch('cloudinit.net.util', 'm_util') diff --git a/cloudinit/sources/tests/test_oracle.py b/cloudinit/sources/tests/test_oracle.py index abf3d359..316efc4f 100644 --- a/cloudinit/sources/tests/test_oracle.py +++ b/cloudinit/sources/tests/test_oracle.py @@ -588,8 +588,6 @@ class TestNetworkConfigFromOpcImds(test_helpers.CiTestCase): class TestNetworkConfigFiltersNetFailover(test_helpers.CiTestCase): - with_logs = True - def setUp(self): super(TestNetworkConfigFiltersNetFailover, self).setUp() self.add_patch(DS_PATH + '.get_interfaces_by_mac', diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index a809fd87..2c6aa44d 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -383,8 +383,6 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): class TestAzureDataSource(CiTestCase): - with_logs = True - def setUp(self): super(TestAzureDataSource, self).setUp() self.tmp = self.tmp_dir() diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 2a81d3f5..41b6c27b 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -158,7 +158,6 @@ class TestMAASDataSource(CiTestCase): @mock.patch("cloudinit.sources.DataSourceMAAS.url_helper.OauthUrlHelper") class TestGetOauthHelper(CiTestCase): - with_logs = True base_cfg = {'consumer_key': 'FAKE_CONSUMER_KEY', 'token_key': 'FAKE_TOKEN_KEY', 'token_secret': 'FAKE_TOKEN_SECRET', diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index 2b22559f..407aa6c4 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -29,8 +29,6 @@ LOG = logging.getLogger(__name__) class TestLocale(t_help.FilesystemMockingTestCase): - with_logs = True - def setUp(self): super(TestLocale, self).setUp() self.new_root = tempfile.mkdtemp() diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py index 1494177d..04aa7d03 100644 --- a/tests/unittests/test_handler/test_handler_puppet.py +++ b/tests/unittests/test_handler/test_handler_puppet.py @@ -16,8 +16,6 @@ LOG = logging.getLogger(__name__) @mock.patch('cloudinit.config.cc_puppet.os') class TestAutostartPuppet(CiTestCase): - with_logs = True - def test_wb_autostart_puppet_updates_puppet_default(self, m_os, m_util): """Update /etc/default/puppet to autostart if it exists.""" diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 9ff17f52..5b2eaa69 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -737,7 +737,6 @@ class TestReadSeeded(helpers.TestCase): class TestSubp(helpers.CiTestCase): - with_logs = True allowed_subp = [BASH, 'cat', helpers.CiTestCase.SUBP_SHELL_TRUE, BOGUS_COMMAND, sys.executable] -- cgit v1.2.3 From 993f3e3e76e56266a83776a8f54dbb3ba59cfce7 Mon Sep 17 00:00:00 2001 From: Gonéri Le Bouder Date: Wed, 25 Mar 2020 13:44:10 -0400 Subject: set_passwords: avoid chpasswd on BSD (#268) Avoid chpasswd on all the BSD variants. --- cloudinit/config/cc_set_passwords.py | 2 +- cloudinit/config/tests/test_set_passwords.py | 16 ++++++++-------- cloudinit/tests/test_util.py | 6 ++++-- cloudinit/util.py | 7 ++++++- 4 files changed, 19 insertions(+), 12 deletions(-) (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 4943d545..7b7aa885 100755 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -241,7 +241,7 @@ def rand_user_password(pwlen=20): def chpasswd(distro, plist_in, hashed=False): - if util.is_FreeBSD(): + if util.is_BSD(): for pentry in plist_in.splitlines(): u, p = pentry.split(":") distro.set_passwd(u, p, hashed=hashed) diff --git a/cloudinit/config/tests/test_set_passwords.py b/cloudinit/config/tests/test_set_passwords.py index 8247c388..2732bd60 100644 --- a/cloudinit/config/tests/test_set_passwords.py +++ b/cloudinit/config/tests/test_set_passwords.py @@ -112,12 +112,12 @@ class TestSetPasswordsHandle(CiTestCase): '\n'.join(valid_hashed_pwds) + '\n')], m_subp.call_args_list) - @mock.patch(MODPATH + "util.is_FreeBSD") + @mock.patch(MODPATH + "util.is_BSD") @mock.patch(MODPATH + "util.subp") - def test_freebsd_calls_custom_pw_cmds_to_set_and_expire_passwords( - self, m_subp, m_is_freebsd): - """FreeBSD calls custom pw commands instead of chpasswd and passwd""" - m_is_freebsd.return_value = True + def test_bsd_calls_custom_pw_cmds_to_set_and_expire_passwords( + self, m_subp, m_is_bsd): + """BSD don't use chpasswd""" + m_is_bsd.return_value = True cloud = self.tmp_cloud(distro='freebsd') valid_pwds = ['ubuntu:passw0rd'] cfg = {'chpasswd': {'list': valid_pwds}} @@ -129,12 +129,12 @@ class TestSetPasswordsHandle(CiTestCase): mock.call(['pw', 'usermod', 'ubuntu', '-p', '01-Jan-1970'])], m_subp.call_args_list) - @mock.patch(MODPATH + "util.is_FreeBSD") + @mock.patch(MODPATH + "util.is_BSD") @mock.patch(MODPATH + "util.subp") def test_handle_on_chpasswd_list_creates_random_passwords(self, m_subp, - m_is_freebsd): + m_is_bsd): """handle parses command set random passwords.""" - m_is_freebsd.return_value = False + m_is_bsd.return_value = False cloud = self.tmp_cloud(distro='ubuntu') valid_random_pwds = [ 'root:R', diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index 877ab5c5..bfccfe1e 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -441,13 +441,15 @@ class TestGetLinuxDistro(CiTestCase): @mock.patch('platform.system') @mock.patch('platform.release') @mock.patch('cloudinit.util._parse_redhat_release') - def test_get_linux_freebsd(self, m_path_exists, m_platform_release, - m_platform_system, m_parse_redhat_release): + def test_get_linux_freebsd(self, m_parse_redhat_release, + m_platform_release, + m_platform_system, m_path_exists): """Verify we get the correct name and release name on FreeBSD.""" m_path_exists.return_value = False m_platform_release.return_value = '12.0-RELEASE-p10' m_platform_system.return_value = 'FreeBSD' m_parse_redhat_release.return_value = {} + util.is_BSD.cache_clear() dist = util.get_linux_distro() self.assertEqual(('freebsd', '12.0-RELEASE-p10', ''), dist) diff --git a/cloudinit/util.py b/cloudinit/util.py index 89889459..db60b9d2 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -542,6 +542,11 @@ def is_ipv4(instr): return len(toks) == 4 +@lru_cache() +def is_BSD(): + return 'BSD' in platform.system() + + @lru_cache() def is_FreeBSD(): return system_info()['variant'] == "freebsd" @@ -625,7 +630,7 @@ def get_linux_distro(): flavor = match.groupdict()['codename'] if distro_name == 'rhel': distro_name = 'redhat' - elif 'BSD' in platform.system(): + elif is_BSD(): distro_name = platform.system().lower() distro_version = platform.release() else: -- cgit v1.2.3 From 9d7b35ce23aaf8741dd49b16e359c96591be3c76 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 15 Apr 2020 16:53:08 -0400 Subject: cc_mounts: fix incorrect format specifiers (#316) LP: #1872836 --- cloudinit/config/cc_mounts.py | 8 ++++---- cloudinit/config/tests/test_mounts.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 cloudinit/config/tests/test_mounts.py (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 1f077bf5..85a89cd1 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -236,20 +236,20 @@ def suggested_swapsize(memsize=None, maxsize=None, fsys=None): return size -def create_swapfile(fname, size): +def create_swapfile(fname: str, size: str) -> None: """Size is in MiB.""" - errmsg = "Failed to create swapfile '%s' of size %dMB via %s: %s" + errmsg = "Failed to create swapfile '%s' of size %sMB via %s: %s" def create_swap(fname, size, method): LOG.debug("Creating swapfile in '%s' on fstype '%s' using '%s'", fname, fstype, method) if method == "fallocate": - cmd = ['fallocate', '-l', '%dM' % size, fname] + cmd = ['fallocate', '-l', '%sM' % size, fname] elif method == "dd": cmd = ['dd', 'if=/dev/zero', 'of=%s' % fname, 'bs=1M', - 'count=%d' % size] + 'count=%s' % size] try: util.subp(cmd, capture=True) diff --git a/cloudinit/config/tests/test_mounts.py b/cloudinit/config/tests/test_mounts.py new file mode 100644 index 00000000..c7dad61a --- /dev/null +++ b/cloudinit/config/tests/test_mounts.py @@ -0,0 +1,22 @@ +# This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + +from cloudinit.config.cc_mounts import create_swapfile + + +M_PATH = 'cloudinit.config.cc_mounts.' + + +class TestCreateSwapfile: + + @mock.patch(M_PATH + 'util.subp') + def test_happy_path(self, m_subp, tmpdir): + swap_file = tmpdir.join("swap-file") + fname = str(swap_file) + + # Some of the calls to util.subp should create the swap file; this + # roughly approximates that + m_subp.side_effect = lambda *args, **kwargs: swap_file.write('') + + create_swapfile(fname, '') + assert mock.call(['mkswap', fname]) in m_subp.call_args_list -- cgit v1.2.3 From 0c5c736727d0f8e363678fe970d70bb889e3893c Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 16 Apr 2020 11:18:36 -0400 Subject: test_mounts: expand happy path test for both happy paths (#319) --- cloudinit/config/tests/test_mounts.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/tests/test_mounts.py b/cloudinit/config/tests/test_mounts.py index c7dad61a..80b54d0f 100644 --- a/cloudinit/config/tests/test_mounts.py +++ b/cloudinit/config/tests/test_mounts.py @@ -1,6 +1,8 @@ # This file is part of cloud-init. See LICENSE file for license information. from unittest import mock +import pytest + from cloudinit.config.cc_mounts import create_swapfile @@ -9,8 +11,10 @@ M_PATH = 'cloudinit.config.cc_mounts.' class TestCreateSwapfile: + @pytest.mark.parametrize('fstype', ('xfs', 'btrfs', 'ext4', 'other')) + @mock.patch(M_PATH + 'util.get_mount_info') @mock.patch(M_PATH + 'util.subp') - def test_happy_path(self, m_subp, tmpdir): + def test_happy_path(self, m_subp, m_get_mount_info, fstype, tmpdir): swap_file = tmpdir.join("swap-file") fname = str(swap_file) @@ -18,5 +22,7 @@ class TestCreateSwapfile: # roughly approximates that m_subp.side_effect = lambda *args, **kwargs: swap_file.write('') + m_get_mount_info.return_value = (mock.ANY, fstype) + create_swapfile(fname, '') assert mock.call(['mkswap', fname]) in m_subp.call_args_list -- cgit v1.2.3 From 38a7e6e9756fdab31264c0d6e93d20432ed111ac Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 24 Apr 2020 09:26:51 -0400 Subject: cloudinit: drop dependencies on unittest2 and contextlib2 (#322) These libraries provide backports of Python 3's stdlib components to Python 2. As we only support Python 3, we can simply use the stdlib now. This pull request does the following: * removes some unneeded compatibility code for the old spelling of `assertRaisesRegex` * replaces invocations of the Python 2-only `assertItemsEqual` with its new name, `assertCountEqual` * replaces all usage of `unittest2` with `unittest` * replaces all usage of `contextlib2` with `contextlib` * drops `unittest2` and `contextlib2` from requirements files and tox.ini It also rewrites some `test_azure` helpers to use bare asserts. We were seeing a strange error in xenial builds of this branch which appear to be stemming from the AssertionError that pytest produces being _different_ from the standard AssertionError. This means that the modified helpers weren't behaving correctly, because they weren't catching AssertionErrors as one would expect. (I believe this is related, in some way, to https://github.com/pytest-dev/pytest/issues/645, but the only version of pytest where we're affected is so far in the past that it's not worth pursuing it any further as we have a workaround.) --- cloudinit/config/tests/test_users_groups.py | 10 ++++---- cloudinit/net/tests/test_dhcp.py | 10 ++++---- cloudinit/net/tests/test_init.py | 2 +- cloudinit/sources/tests/test_init.py | 10 ++++---- cloudinit/tests/helpers.py | 28 ++++++---------------- integration-requirements.txt | 1 - packages/pkg-deps.json | 3 --- test-requirements.txt | 2 -- tests/cloud_tests/testcases/__init__.py | 8 +++---- tests/cloud_tests/testcases/base.py | 12 +++++----- tests/cloud_tests/testcases/modules/ntp_chrony.py | 4 ++-- tests/cloud_tests/verify.py | 4 ++-- tests/unittests/test_builtin_handlers.py | 6 ++--- tests/unittests/test_datasource/test_azure.py | 8 +++---- .../unittests/test_datasource/test_azure_helper.py | 6 ++--- tests/unittests/test_datasource/test_smartos.py | 12 +++++----- .../test_handler/test_handler_ca_certs.py | 6 +---- tests/unittests/test_handler/test_handler_chef.py | 8 +++---- .../test_handler/test_handler_growpart.py | 6 +---- tests/unittests/test_handler/test_schema.py | 4 ++-- tox.ini | 2 -- 21 files changed, 61 insertions(+), 91 deletions(-) (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/tests/test_users_groups.py b/cloudinit/config/tests/test_users_groups.py index f620b597..df89ddb3 100644 --- a/cloudinit/config/tests/test_users_groups.py +++ b/cloudinit/config/tests/test_users_groups.py @@ -39,7 +39,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -65,7 +65,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='freebsd', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_fbsd_user.call_args_list, [mock.call('freebsd', groups='wheel', lock_passwd=True, shell='/bin/tcsh'), @@ -86,7 +86,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -107,7 +107,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -146,7 +146,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index c3fa1e04..bc7bef45 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -62,7 +62,7 @@ class TestParseDHCPLeasesFile(CiTestCase): {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) class TestDHCPRFC3442(CiTestCase): @@ -88,7 +88,7 @@ class TestDHCPRFC3442(CiTestCase): 'renew': '4 2017/07/27 18:02:30', 'expire': '5 2017/07/28 07:08:15'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) def test_parse_lease_finds_classless_static_routes(self): """ @@ -114,7 +114,7 @@ class TestDHCPRFC3442(CiTestCase): 'renew': '4 2017/07/27 18:02:30', 'expire': '5 2017/07/28 07:08:15'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') @@ -324,7 +324,7 @@ class TestDHCPDiscoveryClean(CiTestCase): """) write_file(self.tmp_path('dhcp.leases', tmpdir), lease_content) - self.assertItemsEqual( + self.assertCountEqual( [{'interface': 'eth9', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], dhcp_discovery(dhclient_script, 'eth9', tmpdir)) @@ -389,7 +389,7 @@ class TestDHCPDiscoveryClean(CiTestCase): write_file(pid_file, "%d\n" % my_pid) m_getppid.return_value = 1 # Indicate that dhclient has daemonized - self.assertItemsEqual( + self.assertCountEqual( [{'interface': 'eth9', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], dhcp_discovery(dhclient_script, 'eth9', tmpdir)) diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index 32e70b4c..835ed807 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -397,7 +397,7 @@ class TestGetDeviceList(CiTestCase): """get_devicelist returns a directory listing for SYS_CLASS_NET.""" write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), 'up') write_file(os.path.join(self.sysdir, 'eth1', 'operstate'), 'up') - self.assertItemsEqual(['eth0', 'eth1'], net.get_devicelist()) + self.assertCountEqual(['eth0', 'eth1'], net.get_devicelist()) class TestGetInterfaceMAC(CiTestCase): diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py index 5b3648e9..5b6f1b3f 100644 --- a/cloudinit/sources/tests/test_init.py +++ b/cloudinit/sources/tests/test_init.py @@ -350,7 +350,7 @@ class TestDataSource(CiTestCase): 'region': 'myregion', 'some': {'security-credentials': { 'cred1': 'sekret', 'cred2': 'othersekret'}}}) - self.assertItemsEqual( + self.assertCountEqual( ('merged_cfg', 'security-credentials',), datasource.sensitive_metadata_keys) sys_info = { @@ -401,7 +401,7 @@ class TestDataSource(CiTestCase): 'region': 'myregion', 'some': {'security-credentials': REDACT_SENSITIVE_VALUE}}} } - self.assertItemsEqual(expected, redacted) + self.assertCountEqual(expected, redacted) file_stat = os.stat(json_file) self.assertEqual(0o644, stat.S_IMODE(file_stat.st_mode)) @@ -426,7 +426,7 @@ class TestDataSource(CiTestCase): "x86_64"], "variant": "ubuntu", "dist": ["ubuntu", "20.04", "focal"]} - self.assertItemsEqual( + self.assertCountEqual( ('merged_cfg', 'security-credentials',), datasource.sensitive_metadata_keys) with mock.patch("cloudinit.util.system_info", return_value=sys_info): @@ -476,7 +476,7 @@ class TestDataSource(CiTestCase): 'security-credentials': {'cred1': 'sekret', 'cred2': 'othersekret'}}}} } - self.assertItemsEqual(expected, util.load_json(content)) + self.assertCountEqual(expected, util.load_json(content)) file_stat = os.stat(sensitive_json_file) self.assertEqual(0o600, stat.S_IMODE(file_stat.st_mode)) self.assertEqual(expected, util.load_json(content)) @@ -542,7 +542,7 @@ class TestDataSource(CiTestCase): json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp) content = util.load_file(json_file) instance_json = util.load_json(content) - self.assertItemsEqual( + self.assertCountEqual( ['ds/meta_data/key2/key2.1'], instance_json['base64_encoded_keys']) self.assertEqual( diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index f4db5827..477e14c2 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -13,15 +13,10 @@ import string import sys import tempfile import time +import unittest +from contextlib import ExitStack, contextmanager from unittest import mock - -import unittest2 -from unittest2.util import strclass - -try: - from contextlib import ExitStack, contextmanager -except ImportError: - from contextlib2 import ExitStack, contextmanager +from unittest.util import strclass from cloudinit.config.schema import ( SchemaValidationError, validate_cloudconfig_schema) @@ -35,8 +30,8 @@ from cloudinit import util _real_subp = util.subp # Used for skipping tests -SkipTest = unittest2.SkipTest -skipIf = unittest2.skipIf +SkipTest = unittest.SkipTest +skipIf = unittest.skipIf # Makes the old path start @@ -73,7 +68,7 @@ def retarget_many_wrapper(new_base, am, old_func): return wrapper -class TestCase(unittest2.TestCase): +class TestCase(unittest.TestCase): def reset_global_state(self): """Reset any global state to its original settings. @@ -372,7 +367,7 @@ class HttprettyTestCase(CiTestCase): super(HttprettyTestCase, self).tearDown() -class SchemaTestCaseMixin(unittest2.TestCase): +class SchemaTestCaseMixin(unittest.TestCase): def assertSchemaValid(self, cfg, msg="Valid Schema failed validation."): """Assert the config is valid per self.schema. @@ -504,13 +499,4 @@ if not hasattr(mock.Mock, 'assert_not_called'): raise AssertionError(msg) mock.Mock.assert_not_called = __mock_assert_not_called - -# older unittest2.TestCase (centos6) have only the now-deprecated -# assertRaisesRegexp. Simple assignment makes pylint complain, about -# users of assertRaisesRegex so we use getattr to trick it. -# https://github.com/PyCQA/pylint/issues/1946 -if not hasattr(unittest2.TestCase, 'assertRaisesRegex'): - unittest2.TestCase.assertRaisesRegex = ( - getattr(unittest2.TestCase, 'assertRaisesRegexp')) - # vi: ts=4 expandtab diff --git a/integration-requirements.txt b/integration-requirements.txt index 897d6110..44e45c1b 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -5,7 +5,6 @@ # the packages/pkg-deps.json file as well. # -unittest2 # ec2 backend boto3==1.5.9 diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json index d09ec33f..f02e8348 100644 --- a/packages/pkg-deps.json +++ b/packages/pkg-deps.json @@ -10,9 +10,6 @@ "2" : "python-yaml", "3" : "python3-yaml" }, - "contextlib2" : { - "2" : "python-contextlib2" - }, "pyserial" : { "2" : "python-serial", "3" : "python3-serial" diff --git a/test-requirements.txt b/test-requirements.txt index 057115b6..0a6a04d4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,9 +1,7 @@ # Needed generally in tests httpretty>=0.7.1 pytest -unittest2 pytest-cov # Only really needed on older versions of python -contextlib2 setuptools diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py index 6bb39f77..e8c371ca 100644 --- a/tests/cloud_tests/testcases/__init__.py +++ b/tests/cloud_tests/testcases/__init__.py @@ -4,7 +4,7 @@ import importlib import inspect -import unittest2 +import unittest from cloudinit.util import read_conf @@ -48,7 +48,7 @@ def get_test_class(test_name, test_data, test_conf): def __str__(self): return "%s (%s)" % (self._testMethodName, - unittest2.util.strclass(self._realclass)) + unittest.util.strclass(self._realclass)) @classmethod def setUpClass(cls): @@ -62,9 +62,9 @@ def get_suite(test_name, data, conf): @return_value: a test suite """ - suite = unittest2.TestSuite() + suite = unittest.TestSuite() suite.addTest( - unittest2.defaultTestLoader.loadTestsFromTestCase( + unittest.defaultTestLoader.loadTestsFromTestCase( get_test_class(test_name, data, conf))) return suite diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index b0dfc144..7b67f54e 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -5,15 +5,15 @@ import crypt import json import re -import unittest2 +import unittest from cloudinit import util as c_util -SkipTest = unittest2.SkipTest +SkipTest = unittest.SkipTest -class CloudTestCase(unittest2.TestCase): +class CloudTestCase(unittest.TestCase): """Base test class for verifiers.""" # data gets populated in get_suite.setUpClass @@ -172,7 +172,7 @@ class CloudTestCase(unittest2.TestCase): 'Skipping instance-data.json test.' ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) - self.assertItemsEqual(['merged_cfg'], instance_data['sensitive_keys']) + self.assertCountEqual(['merged_cfg'], instance_data['sensitive_keys']) ds = instance_data.get('ds', {}) v1_data = instance_data.get('v1', {}) metadata = ds.get('meta-data', {}) @@ -237,7 +237,7 @@ class CloudTestCase(unittest2.TestCase): ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) v1_data = instance_data.get('v1', {}) - self.assertItemsEqual([], sorted(instance_data['base64_encoded_keys'])) + self.assertCountEqual([], sorted(instance_data['base64_encoded_keys'])) self.assertEqual('unknown', v1_data['cloud_name']) self.assertEqual('lxd', v1_data['platform']) self.assertEqual( @@ -291,7 +291,7 @@ class CloudTestCase(unittest2.TestCase): ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) v1_data = instance_data.get('v1', {}) - self.assertItemsEqual([], instance_data['base64_encoded_keys']) + self.assertCountEqual([], instance_data['base64_encoded_keys']) self.assertEqual('unknown', v1_data['cloud_name']) self.assertEqual('nocloud', v1_data['platform']) subplatform = v1_data['subplatform'] diff --git a/tests/cloud_tests/testcases/modules/ntp_chrony.py b/tests/cloud_tests/testcases/modules/ntp_chrony.py index 0f4c3d08..7d341773 100644 --- a/tests/cloud_tests/testcases/modules/ntp_chrony.py +++ b/tests/cloud_tests/testcases/modules/ntp_chrony.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. """cloud-init Integration Test Verify Script.""" -import unittest2 +import unittest from tests.cloud_tests.testcases import base @@ -13,7 +13,7 @@ class TestNtpChrony(base.CloudTestCase): """Skip this suite of tests on lxd and artful or older.""" if self.platform == 'lxd': if self.is_distro('ubuntu') and self.os_version_cmp('artful') <= 0: - raise unittest2.SkipTest( + raise unittest.SkipTest( 'No support for chrony on containers <= artful.' ' LP: #1589780') return super(TestNtpChrony, self).setUp() diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py index 7018f4d5..0295af40 100644 --- a/tests/cloud_tests/verify.py +++ b/tests/cloud_tests/verify.py @@ -3,7 +3,7 @@ """Verify test results.""" import os -import unittest2 +import unittest from tests.cloud_tests import (config, LOG, util, testcases) @@ -18,7 +18,7 @@ def verify_data(data_dir, platform, os_name, tests): @return_value: {: {passed: True/False, failures: []}} """ base_dir = os.sep.join((data_dir, platform, os_name)) - runner = unittest2.TextTestRunner(verbosity=util.current_verbosity()) + runner = unittest.TextTestRunner(verbosity=util.current_verbosity()) res = {} for test_name in tests: LOG.debug('verifying test data for %s', test_name) diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index b92ffc79..9045e743 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -109,7 +109,7 @@ class TestJinjaTemplatePartHandler(CiTestCase): cloudconfig_handler = CloudConfigPartHandler(self.paths) h = JinjaTemplatePartHandler( self.paths, sub_handlers=[script_handler, cloudconfig_handler]) - self.assertItemsEqual( + self.assertCountEqual( ['text/cloud-config', 'text/cloud-config-jsonp', 'text/x-shellscript'], h.sub_handlers) @@ -120,7 +120,7 @@ class TestJinjaTemplatePartHandler(CiTestCase): cloudconfig_handler = CloudConfigPartHandler(self.paths) h = JinjaTemplatePartHandler( self.paths, sub_handlers=[script_handler, cloudconfig_handler]) - self.assertItemsEqual( + self.assertCountEqual( ['text/cloud-config', 'text/cloud-config-jsonp', 'text/x-shellscript'], h.sub_handlers) @@ -302,7 +302,7 @@ class TestConvertJinjaInstanceData(CiTestCase): expected_data.update({'v1key1': 'v1.1', 'v2key1': 'v2.1'}) converted_data = convert_jinja_instance_data(data=data) - self.assertItemsEqual( + self.assertCountEqual( ['ds', 'v1', 'v2', 'v1key1', 'v2key1'], converted_data.keys()) self.assertEqual( expected_data, diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 2c6aa44d..2f8561cb 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -528,14 +528,14 @@ scbus-1 on xpt0 bus 0 def tags_exists(x, y): for tag in x.keys(): - self.assertIn(tag, y) + assert tag in y for tag in y.keys(): - self.assertIn(tag, x) + assert tag in x def tags_equal(x, y): for x_val in x.values(): y_val = y.get(x_val.tag) - self.assertEqual(x_val.text, y_val.text) + assert x_val.text == y_val.text old_cnt = create_tag_index(oxml) new_cnt = create_tag_index(nxml) @@ -649,7 +649,7 @@ scbus-1 on xpt0 bus 0 crawled_metadata = dsrc.crawl_metadata() - self.assertItemsEqual( + self.assertCountEqual( crawled_metadata.keys(), ['cfg', 'files', 'metadata', 'userdata_raw']) self.assertEqual(crawled_metadata['cfg'], expected_cfg) diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 007df09f..6344b0bb 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import os -import unittest2 +import unittest from textwrap import dedent from cloudinit.sources.helpers import azure as azure_helper @@ -332,7 +332,7 @@ class TestOpenSSLManagerActions(CiTestCase): path = 'tests/data/azure' return os.path.join(path, name) - @unittest2.skip("todo move to cloud_test") + @unittest.skip("todo move to cloud_test") def test_pubkey_extract(self): cert = load_file(self._data_file('pubkey_extract_cert')) good_key = load_file(self._data_file('pubkey_extract_ssh_key')) @@ -344,7 +344,7 @@ class TestOpenSSLManagerActions(CiTestCase): fingerprint = sslmgr._get_fingerprint_from_cert(cert) self.assertEqual(good_fingerprint, fingerprint) - @unittest2.skip("todo move to cloud_test") + @unittest.skip("todo move to cloud_test") @mock.patch.object(azure_helper.OpenSSLManager, '_decrypt_certs_from_xml') def test_parse_certificates(self, mock_decrypt_certs): """Azure control plane puts private keys as well as certificates diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 62084de5..0f0b42bb 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -22,7 +22,7 @@ import os.path import re import signal import stat -import unittest2 +import unittest import uuid from cloudinit import serial @@ -1095,11 +1095,11 @@ class TestNetworkConversion(CiTestCase): self.assertEqual(expected, found) -@unittest2.skipUnless(get_smartos_environ() == SMARTOS_ENV_KVM, - "Only supported on KVM and bhyve guests under SmartOS") -@unittest2.skipUnless(os.access(SERIAL_DEVICE, os.W_OK), - "Requires write access to " + SERIAL_DEVICE) -@unittest2.skipUnless(HAS_PYSERIAL is True, "pyserial not available") +@unittest.skipUnless(get_smartos_environ() == SMARTOS_ENV_KVM, + "Only supported on KVM and bhyve guests under SmartOS") +@unittest.skipUnless(os.access(SERIAL_DEVICE, os.W_OK), + "Requires write access to " + SERIAL_DEVICE) +@unittest.skipUnless(HAS_PYSERIAL is True, "pyserial not available") class TestSerialConcurrency(CiTestCase): """ This class tests locking on an actual serial port, and as such can only diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index 5b4105dd..286ef771 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -11,13 +11,9 @@ import logging import shutil import tempfile import unittest +from contextlib import ExitStack from unittest import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - class TestNoConfig(unittest.TestCase): def setUp(self): diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 2dab3a54..513c18b5 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -65,18 +65,18 @@ class TestInstallChefOmnibus(HttprettyTestCase): cc_chef.install_chef_from_omnibus() expected_kwargs = {'retries': cc_chef.OMNIBUS_URL_RETRIES, 'url': cc_chef.OMNIBUS_URL} - self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[0][1]) + self.assertCountEqual(expected_kwargs, m_rdurl.call_args_list[0][1]) cc_chef.install_chef_from_omnibus(retries=10) expected_kwargs = {'retries': 10, 'url': cc_chef.OMNIBUS_URL} - self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[1][1]) + self.assertCountEqual(expected_kwargs, m_rdurl.call_args_list[1][1]) expected_subp_kwargs = { 'args': ['-v', '2.0'], 'basename': 'chef-omnibus-install', 'blob': m_rdurl.return_value.contents, 'capture': False } - self.assertItemsEqual( + self.assertCountEqual( expected_subp_kwargs, m_subp_blob.call_args_list[0][1]) @@ -97,7 +97,7 @@ class TestInstallChefOmnibus(HttprettyTestCase): 'blob': response, 'capture': False } - self.assertItemsEqual(expected_kwargs, called_kwargs) + self.assertCountEqual(expected_kwargs, called_kwargs) class TestChef(FilesystemMockingTestCase): diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index 43b53745..501bcca5 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -11,13 +11,9 @@ import logging import os import re import unittest +from contextlib import ExitStack from unittest import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - # growpart: # mode: auto # off, on, auto, 'growpart' # devices: ['root'] diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index abde02d3..98628120 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -20,7 +20,7 @@ class GetSchemaTest(CiTestCase): def test_get_schema_coalesces_known_schema(self): """Every cloudconfig module with schema is listed in allOf keyword.""" schema = get_schema() - self.assertItemsEqual( + self.assertCountEqual( [ 'cc_bootcmd', 'cc_ntp', @@ -38,7 +38,7 @@ class GetSchemaTest(CiTestCase): schema['$schema']) # FULL_SCHEMA is updated by the get_schema call from cloudinit.config.schema import FULL_SCHEMA - self.assertItemsEqual(['id', '$schema', 'allOf'], FULL_SCHEMA.keys()) + self.assertCountEqual(['id', '$schema', 'allOf'], FULL_SCHEMA.keys()) def test_get_schema_returns_global_when_set(self): """When FULL_SCHEMA global is already set, get_schema returns it.""" diff --git a/tox.ini b/tox.ini index 416ddcb8..906519c4 100644 --- a/tox.ini +++ b/tox.ini @@ -75,8 +75,6 @@ deps = # test-requirements httpretty==0.9.6 mock==1.3.0 - unittest2==1.1.0 - contextlib2==0.5.1 [testenv:xenial] # When updating this commands definition, also update the definition in -- cgit v1.2.3 From 09492b1d0f4b1826a7a98709c61d48bf14a1ec64 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 14 May 2020 15:39:19 -0400 Subject: cloudinit: minor pylint fixes (#360) We recently discovered that pylint is failing to report some errors when invoked across our entire codebase (see https://github.com/PyCQA/pylint/issues/3611). I've run pylint across every Python file under cloudinit/[0], and this commit fixes the issues so-discovered. [0] find cloudinit/ -name "*.py" | xargs -n 1 -t .tox/pylint/bin/python -m pylint --- cloudinit/config/tests/test_snap.py | 2 +- cloudinit/config/tests/test_ubuntu_drivers.py | 7 +++++++ cloudinit/sources/helpers/tests/test_netlink.py | 2 +- cloudinit/tests/test_conftest.py | 2 ++ 4 files changed, 11 insertions(+), 2 deletions(-) (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py index cbbb173d..70676942 100644 --- a/cloudinit/config/tests/test_snap.py +++ b/cloudinit/config/tests/test_snap.py @@ -345,7 +345,7 @@ class TestSchema(CiTestCase, SchemaTestCaseMixin): def test_duplicates_are_fine_array_array(self): """Duplicated commands array/array entries are allowed.""" self.assertSchemaValid( - {'commands': [["echo", "bye"], ["echo" "bye"]]}, + {'commands': [["echo", "bye"], ["echo", "bye"]]}, "command entries can be duplicate.") def test_duplicates_are_fine_array_string(self): diff --git a/cloudinit/config/tests/test_ubuntu_drivers.py b/cloudinit/config/tests/test_ubuntu_drivers.py index 46952692..0aec1265 100644 --- a/cloudinit/config/tests/test_ubuntu_drivers.py +++ b/cloudinit/config/tests/test_ubuntu_drivers.py @@ -16,6 +16,13 @@ OLD_UBUNTU_DRIVERS_ERROR_STDERR = ( "(choose from 'list', 'autoinstall', 'devices', 'debug')\n") +# The tests in this module call helper methods which are decorated with +# mock.patch. pylint doesn't understand that mock.patch passes parameters to +# the decorated function, so it incorrectly reports that we aren't passing +# values for all parameters. Instead of annotating every single call, we +# disable it for the entire module: +# pylint: disable=no-value-for-parameter + class AnyTempScriptAndDebconfFile(object): def __init__(self, tmp_dir, debconf_file): diff --git a/cloudinit/sources/helpers/tests/test_netlink.py b/cloudinit/sources/helpers/tests/test_netlink.py index c2898a16..58c3adc6 100644 --- a/cloudinit/sources/helpers/tests/test_netlink.py +++ b/cloudinit/sources/helpers/tests/test_netlink.py @@ -87,7 +87,7 @@ class TestParseNetlinkMessage(CiTestCase): data = None with self.assertRaises(AssertionError) as context: read_rta_oper_state(data) - self.assertTrue('data is none', str(context.exception)) + self.assertEqual('data is none', str(context.exception)) def test_read_invalid_rta_operstate_none(self): '''read_rta_oper_state returns none if operstate is none''' diff --git a/cloudinit/tests/test_conftest.py b/cloudinit/tests/test_conftest.py index 11b91327..b39637d8 100644 --- a/cloudinit/tests/test_conftest.py +++ b/cloudinit/tests/test_conftest.py @@ -13,6 +13,8 @@ class TestDisableSubpUsage: def test_typeerrors_on_incorrect_usage(self): with pytest.raises(TypeError): + # We are intentionally passing no value for a parameter, so: + # pylint: disable=no-value-for-parameter util.subp() @pytest.mark.parametrize('disable_subp_usage', [False], indirect=True) -- cgit v1.2.3 From 8bcf1c069d8ddf28898342e541254bdeb54445df Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 18 May 2020 13:30:27 -0400 Subject: cc_snap: apply validation to snap.commands properties (#364) Specifically, ensure that given values are either strings, or arrays of strings. --- cloudinit/config/cc_snap.py | 14 ++++++++++++++ cloudinit/config/tests/test_snap.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py index fbe7787a..3bf2e250 100644 --- a/cloudinit/config/cc_snap.py +++ b/cloudinit/config/cc_snap.py @@ -85,6 +85,14 @@ schema = { 01: ['snap', 'install', 'vlc'] 02: snap install vlc 03: 'snap install vlc' + """), dedent("""\ + # You can use a list of commands + snap: + commands: + - ['install', 'vlc'] + - ['snap', 'install', 'vlc'] + - snap install vlc + - 'snap install vlc' """)], 'frequency': PER_INSTANCE, 'type': 'object', @@ -110,6 +118,12 @@ schema = { 'additionalItems': False, # Reject non-string & non-list 'minItems': 1, 'minProperties': 1, + 'additionalProperties': { + 'oneOf': [ + {'type': 'string'}, + {'type': 'array', 'items': {'type': 'string'}}, + ], + }, }, 'squashfuse_in_container': { 'type': 'boolean' diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py index 70676942..2be30186 100644 --- a/cloudinit/config/tests/test_snap.py +++ b/cloudinit/config/tests/test_snap.py @@ -310,6 +310,38 @@ class TestSchema(CiTestCase, SchemaTestCaseMixin): {'snap': {'commands': {'01': 'also valid'}}}, schema) self.assertEqual('', self.logs.getvalue()) + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_when_commands_values_are_invalid_type(self, _): + """Warnings when snap:commands values are invalid type (e.g. int)""" + validate_cloudconfig_schema( + {'snap': {'commands': [123]}}, schema) + validate_cloudconfig_schema( + {'snap': {'commands': {'01': 123}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\n" + "snap.commands.0: 123 is not valid under any of the given" + " schemas\n" + "WARNING: Invalid config:\n" + "snap.commands.01: 123 is not valid under any of the given" + " schemas\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_when_commands_list_values_are_invalid_type(self, _): + """Warnings when snap:commands list values are wrong type (e.g. int)""" + validate_cloudconfig_schema( + {'snap': {'commands': [["snap", "install", 123]]}}, schema) + validate_cloudconfig_schema( + {'snap': {'commands': {'01': ["snap", "install", 123]}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\n" + "snap.commands.0: ['snap', 'install', 123] is not valid under any" + " of the given schemas\n", + "WARNING: Invalid config:\n" + "snap.commands.0: ['snap', 'install', 123] is not valid under any" + " of the given schemas\n", + self.logs.getvalue()) + @mock.patch('cloudinit.config.cc_snap.add_assertions') def test_warn_schema_assertions_is_not_list_or_dict(self, _): """Warn when snap:assertions config is not a list or dict.""" -- cgit v1.2.3 From fae90f14f0668a673e02bc7313bc84828e7cae71 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 19 May 2020 22:52:18 -0400 Subject: test_resolv_conf: refresh stale comment (#374) --- cloudinit/config/tests/test_resolv_conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/tests/test_resolv_conf.py b/cloudinit/config/tests/test_resolv_conf.py index f3d97b82..6546a0b5 100644 --- a/cloudinit/config/tests/test_resolv_conf.py +++ b/cloudinit/config/tests/test_resolv_conf.py @@ -35,8 +35,8 @@ class TestGenerateResolvConf: # Patch in templater so we can assert on the actual generated content @mock.patch("cloudinit.templater.util.write_file") # Parameterise with the value to be passed to generate_resolv_conf as the - # `params` parameter, and a list of the expected lines after the header as - # `extra_lines`. + # params parameter, and the expected line after the header as + # expected_extra_line. @pytest.mark.parametrize( "params,expected_extra_line", [ -- cgit v1.2.3 From 98a6d9a0da82a764943c9495d7f21f3acc5ccd05 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 21 May 2020 11:16:35 -0400 Subject: cc_snap: validate that assertions property values are strings (#370) And add an example of providing a list of assertions. --- cloudinit/config/cc_snap.py | 14 +++++++++----- cloudinit/config/tests/test_snap.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py index 3bf2e250..8178562e 100644 --- a/cloudinit/config/cc_snap.py +++ b/cloudinit/config/cc_snap.py @@ -93,6 +93,13 @@ schema = { - ['snap', 'install', 'vlc'] - snap install vlc - 'snap install vlc' + """), dedent("""\ + # You can use a list of assertions + snap: + assertions: + - signed_assertion_blob_here + - | + signed_assertion_blob_here """)], 'frequency': PER_INSTANCE, 'type': 'object', @@ -106,7 +113,8 @@ schema = { 'additionalItems': False, # Reject items non-string 'minItems': 1, 'minProperties': 1, - 'uniqueItems': True + 'uniqueItems': True, + 'additionalProperties': {'type': 'string'}, }, 'commands': { 'type': ['object', 'array'], # Array of strings or dict @@ -136,10 +144,6 @@ schema = { } } -# TODO schema for 'assertions' and 'commands' are too permissive at the moment. -# Once python-jsonschema supports schema draft 6 add support for arbitrary -# object keys with 'patternProperties' constraint to validate string values. - __doc__ = get_schema_doc(schema) # Supplement python help() SNAP_CMD = "snap" diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py index 2be30186..95270fa0 100644 --- a/cloudinit/config/tests/test_snap.py +++ b/cloudinit/config/tests/test_snap.py @@ -342,6 +342,20 @@ class TestSchema(CiTestCase, SchemaTestCaseMixin): " of the given schemas\n", self.logs.getvalue()) + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_when_assertions_values_are_invalid_type(self, _): + """Warnings when snap:assertions values are invalid type (e.g. int)""" + validate_cloudconfig_schema( + {'snap': {'assertions': [123]}}, schema) + validate_cloudconfig_schema( + {'snap': {'assertions': {'01': 123}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\n" + "snap.assertions.0: 123 is not of type 'string'\n" + "WARNING: Invalid config:\n" + "snap.assertions.01: 123 is not of type 'string'\n", + self.logs.getvalue()) + @mock.patch('cloudinit.config.cc_snap.add_assertions') def test_warn_schema_assertions_is_not_list_or_dict(self, _): """Warn when snap:assertions config is not a list or dict.""" -- cgit v1.2.3 From fc07d633f7cb694423349a2c4b10c91c4b4981a2 Mon Sep 17 00:00:00 2001 From: Matthew Ruffell Date: Tue, 2 Jun 2020 09:55:40 +1200 Subject: cc_grub_dpkg: determine idevs in more robust manner with grub-probe (#358) Replace the hardcoded list of devices with a more robust way of determining the device which grub is installed to. We use grub-probe to fetch the underlying disk the /boot directory is located on, and attempt to match the disk with its /dev/disk/by-id value. If no such /dev/disk/by-id/ value exists, we fallback to the plain disk name. The changes are robust to unstable kernel device names and ordering, and use /dev/disk/by-id values to populate grub-pc/install_devices where possible. LP: #1877491 --- cloudinit/config/cc_grub_dpkg.py | 95 ++++++++++++----- cloudinit/config/tests/test_grub_dpkg.py | 176 +++++++++++++++++++++++++++++++ tools/.github-cla-signers | 1 + 3 files changed, 247 insertions(+), 25 deletions(-) create mode 100644 cloudinit/config/tests/test_grub_dpkg.py (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index a323edfa..7888464e 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -1,8 +1,9 @@ -# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2009-2010, 2020 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser # Author: Juerg Haefliger +# Author: Matthew Ruffell # # This file is part of cloud-init. See LICENSE file for license information. @@ -15,15 +16,15 @@ Configure which device is used as the target for grub installation. This module should work correctly by default without any user configuration. It can be enabled/disabled using the ``enabled`` config key in the ``grub_dpkg`` config dict. The global config key ``grub-dpkg`` is an alias for ``grub_dpkg``. If no -installation device is specified this module will look for the first existing -device in: +installation device is specified this module will execute grub-probe to +determine which disk the /boot directory is associated with. - - ``/dev/sda`` - - ``/dev/vda`` - - ``/dev/xvda`` - - ``/dev/sda1`` - - ``/dev/vda1`` - - ``/dev/xvda1`` +The value which is placed into the debconf database is in the format which the +grub postinstall script expects. Normally, this is a /dev/disk/by-id/ value, +but we do fallback to the plain disk name if a by-id name is not present. + +If this module is executed inside a container, then the debconf database is +seeded with empty values, and install_devices_empty is set to true. **Internal name:** ``cc_grub_dpkg`` @@ -43,10 +44,66 @@ device in: import os from cloudinit import util +from cloudinit.util import ProcessExecutionError distros = ['ubuntu', 'debian'] +def fetch_idevs(log): + """ + Fetches the /dev/disk/by-id device grub is installed to. + Falls back to plain disk name if no by-id entry is present. + """ + disk = "" + devices = [] + + try: + # get the root disk where the /boot directory resides. + disk = util.subp(['grub-probe', '-t', 'disk', '/boot'], + capture=True)[0].strip() + except ProcessExecutionError as e: + # grub-common may not be installed, especially on containers + # FileNotFoundError is a nested exception of ProcessExecutionError + if isinstance(e.reason, FileNotFoundError): + log.debug("'grub-probe' not found in $PATH") + # disks from the container host are present in /proc and /sys + # which is where grub-probe determines where /boot is. + # it then checks for existence in /dev, which fails as host disks + # are not exposed to the container. + elif "failed to get canonical path" in e.stderr: + log.debug("grub-probe 'failed to get canonical path'") + else: + # something bad has happened, continue to log the error + raise + except Exception: + util.logexc(log, "grub-probe failed to execute for grub-dpkg") + + if not disk or not os.path.exists(disk): + # If we failed to detect a disk, we can return early + return '' + + try: + # check if disk exists and use udevadm to fetch symlinks + devices = util.subp( + ['udevadm', 'info', '--root', '--query=symlink', disk], + capture=True + )[0].strip().split() + except Exception: + util.logexc( + log, "udevadm DEVLINKS symlink query failed for disk='%s'", disk + ) + + log.debug('considering these device symlinks: %s', ','.join(devices)) + # filter symlinks for /dev/disk/by-id entries + devices = [dev for dev in devices if 'disk/by-id' in dev] + log.debug('filtered to these disk/by-id symlinks: %s', ','.join(devices)) + # select first device if there is one, else fall back to plain name + idevs = sorted(devices)[0] if devices else disk + log.debug('selected %s', idevs) + + return idevs + + def handle(name, cfg, _cloud, log, _args): mycfg = cfg.get("grub_dpkg", cfg.get("grub-dpkg", {})) @@ -62,22 +119,10 @@ def handle(name, cfg, _cloud, log, _args): idevs_empty = util.get_cfg_option_str( mycfg, "grub-pc/install_devices_empty", None) - if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): - if idevs is None: - idevs = "" - if idevs_empty is None: - idevs_empty = "true" - else: - if idevs_empty is None: - idevs_empty = "false" - if idevs is None: - idevs = "/dev/sda" - for dev in ("/dev/sda", "/dev/vda", "/dev/xvda", - "/dev/sda1", "/dev/vda1", "/dev/xvda1"): - if os.path.exists(dev): - idevs = dev - break + if idevs is None: + idevs = fetch_idevs(log) + if idevs_empty is None: + idevs_empty = "false" if idevs else "true" # now idevs and idevs_empty are set to determined values # or, those set by user diff --git a/cloudinit/config/tests/test_grub_dpkg.py b/cloudinit/config/tests/test_grub_dpkg.py new file mode 100644 index 00000000..01efa330 --- /dev/null +++ b/cloudinit/config/tests/test_grub_dpkg.py @@ -0,0 +1,176 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import pytest + +from unittest import mock +from logging import Logger +from cloudinit.util import ProcessExecutionError +from cloudinit.config.cc_grub_dpkg import fetch_idevs, handle + + +class TestFetchIdevs: + """Tests cc_grub_dpkg.fetch_idevs()""" + + # Note: udevadm info returns devices in a large single line string + @pytest.mark.parametrize( + "grub_output,path_exists,expected_log_call,udevadm_output" + ",expected_idevs", + [ + # Inside a container, grub not installed + ( + ProcessExecutionError(reason=FileNotFoundError()), + False, + mock.call("'grub-probe' not found in $PATH"), + '', + '', + ), + # Inside a container, grub installed + ( + ProcessExecutionError(stderr="failed to get canonical path"), + False, + mock.call("grub-probe 'failed to get canonical path'"), + '', + '', + ), + # KVM Instance + ( + ['/dev/vda'], + True, + None, + ( + '/dev/disk/by-path/pci-0000:00:00.0 ', + '/dev/disk/by-path/virtio-pci-0000:00:00.0 ' + ), + '/dev/vda', + ), + # Xen Instance + ( + ['/dev/xvda'], + True, + None, + '', + '/dev/xvda', + ), + # NVMe Hardware Instance + ( + ['/dev/nvme1n1'], + True, + None, + ( + '/dev/disk/by-id/nvme-Company_hash000 ', + '/dev/disk/by-id/nvme-nvme.000-000-000-000-000 ', + '/dev/disk/by-path/pci-0000:00:00.0-nvme-0 ' + ), + '/dev/disk/by-id/nvme-Company_hash000', + ), + # SCSI Hardware Instance + ( + ['/dev/sda'], + True, + None, + ( + '/dev/disk/by-id/company-user-1 ', + '/dev/disk/by-id/scsi-0Company_user-1 ', + '/dev/disk/by-path/pci-0000:00:00.0-scsi-0:0:0:0 ' + ), + '/dev/disk/by-id/company-user-1', + ), + ], + ) + @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc") + @mock.patch("cloudinit.config.cc_grub_dpkg.os.path.exists") + @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp") + def test_fetch_idevs(self, m_subp, m_exists, m_logexc, grub_output, + path_exists, expected_log_call, udevadm_output, + expected_idevs): + """Tests outputs from grub-probe and udevadm info against grub-dpkg""" + m_subp.side_effect = [ + grub_output, + ["".join(udevadm_output)] + ] + m_exists.return_value = path_exists + log = mock.Mock(spec=Logger) + idevs = fetch_idevs(log) + assert expected_idevs == idevs + if expected_log_call is not None: + assert expected_log_call in log.debug.call_args_list + + +class TestHandle: + """Tests cc_grub_dpkg.handle()""" + + @pytest.mark.parametrize( + "cfg_idevs,cfg_idevs_empty,fetch_idevs_output,expected_log_output", + [ + ( + # No configuration + None, + None, + '/dev/disk/by-id/nvme-Company_hash000', + ( + "Setting grub debconf-set-selections with ", + "'/dev/disk/by-id/nvme-Company_hash000','false'" + ), + ), + ( + # idevs set, idevs_empty unset + '/dev/sda', + None, + '/dev/sda', + ( + "Setting grub debconf-set-selections with ", + "'/dev/sda','false'" + ), + ), + ( + # idevs unset, idevs_empty set + None, + 'true', + '/dev/xvda', + ( + "Setting grub debconf-set-selections with ", + "'/dev/xvda','true'" + ), + ), + ( + # idevs set, idevs_empty set + '/dev/vda', + 'false', + '/dev/disk/by-id/company-user-1', + ( + "Setting grub debconf-set-selections with ", + "'/dev/vda','false'" + ), + ), + ( + # idevs set, idevs_empty set + # Respect what the user defines, even if its logically wrong + '/dev/nvme0n1', + 'true', + '', + ( + "Setting grub debconf-set-selections with ", + "'/dev/nvme0n1','true'" + ), + ) + ], + ) + @mock.patch("cloudinit.config.cc_grub_dpkg.fetch_idevs") + @mock.patch("cloudinit.config.cc_grub_dpkg.util.get_cfg_option_str") + @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc") + @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp") + def test_handle(self, m_subp, m_logexc, m_get_cfg_str, m_fetch_idevs, + cfg_idevs, cfg_idevs_empty, fetch_idevs_output, + expected_log_output): + """Test setting of correct debconf database entries""" + m_get_cfg_str.side_effect = [ + cfg_idevs, + cfg_idevs_empty + ] + m_fetch_idevs.return_value = fetch_idevs_output + log = mock.Mock(spec=Logger) + handle(mock.Mock(), mock.Mock(), mock.Mock(), log, mock.Mock()) + log.debug.assert_called_with("".join(expected_log_output)) + + +# vi: ts=4 expandtab diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 2c924f85..c3113705 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -2,6 +2,7 @@ beezly bipinbachhao dhensby lucasmoura +matthewruffell nishigori tomponline TheRealFalcon -- cgit v1.2.3 From 3c551f6ebc12f7729a2755c89b19b9000e27cc88 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 8 Jun 2020 12:49:12 -0400 Subject: Move subp into its own module. (#416) This was painful, but it finishes a TODO from cloudinit/subp.py. It moves the following from util to subp: ProcessExecutionError subp which target_path I moved subp_blob_in_tempfile into cc_chef, which is its only caller. That saved us from having to deal with it using write_file and temp_utils from subp (which does not import any cloudinit things now). It is arguable that 'target_path' could be moved to a 'path_utils' or something, but in order to use it from subp and also from utils, we had to get it out of utils. --- cloudinit/analyze/dump.py | 3 +- cloudinit/analyze/show.py | 5 +- cloudinit/analyze/tests/test_boot.py | 16 +- cloudinit/analyze/tests/test_dump.py | 3 +- cloudinit/cmd/clean.py | 5 +- cloudinit/cmd/devel/logs.py | 4 +- cloudinit/cmd/devel/tests/test_logs.py | 3 +- cloudinit/config/cc_apt_configure.py | 21 +- cloudinit/config/cc_bootcmd.py | 3 +- cloudinit/config/cc_byobu.py | 3 +- cloudinit/config/cc_ca_certs.py | 5 +- cloudinit/config/cc_chef.py | 35 +- cloudinit/config/cc_disable_ec2_metadata.py | 7 +- cloudinit/config/cc_disk_setup.py | 45 +-- cloudinit/config/cc_emit_upstart.py | 8 +- cloudinit/config/cc_fan.py | 7 +- cloudinit/config/cc_growpart.py | 25 +- cloudinit/config/cc_grub_dpkg.py | 9 +- cloudinit/config/cc_keys_to_console.py | 3 +- cloudinit/config/cc_landscape.py | 3 +- cloudinit/config/cc_lxd.py | 23 +- cloudinit/config/cc_mcollective.py | 3 +- cloudinit/config/cc_mounts.py | 15 +- cloudinit/config/cc_ntp.py | 9 +- .../config/cc_package_update_upgrade_install.py | 3 +- cloudinit/config/cc_power_state_change.py | 3 +- cloudinit/config/cc_puppet.py | 9 +- cloudinit/config/cc_resizefs.py | 9 +- cloudinit/config/cc_rh_subscription.py | 19 +- cloudinit/config/cc_rsyslog.py | 5 +- cloudinit/config/cc_salt_minion.py | 4 +- cloudinit/config/cc_seed_random.py | 5 +- cloudinit/config/cc_set_passwords.py | 5 +- cloudinit/config/cc_snap.py | 7 +- cloudinit/config/cc_spacewalk.py | 8 +- cloudinit/config/cc_ssh.py | 7 +- cloudinit/config/cc_ssh_import_id.py | 5 +- cloudinit/config/cc_ubuntu_advantage.py | 11 +- cloudinit/config/cc_ubuntu_drivers.py | 9 +- .../config/tests/test_disable_ec2_metadata.py | 12 +- cloudinit/config/tests/test_grub_dpkg.py | 6 +- cloudinit/config/tests/test_mounts.py | 4 +- cloudinit/config/tests/test_set_passwords.py | 22 +- cloudinit/config/tests/test_snap.py | 12 +- cloudinit/config/tests/test_ubuntu_advantage.py | 28 +- cloudinit/config/tests/test_ubuntu_drivers.py | 26 +- cloudinit/conftest.py | 28 +- cloudinit/distros/__init__.py | 25 +- cloudinit/distros/arch.py | 17 +- cloudinit/distros/bsd.py | 7 +- cloudinit/distros/debian.py | 9 +- cloudinit/distros/freebsd.py | 13 +- cloudinit/distros/gentoo.py | 17 +- cloudinit/distros/netbsd.py | 11 +- cloudinit/distros/openbsd.py | 3 +- cloudinit/distros/opensuse.py | 7 +- cloudinit/distros/rhel.py | 9 +- cloudinit/gpg.py | 14 +- cloudinit/handlers/boot_hook.py | 5 +- cloudinit/handlers/upstart_job.py | 11 +- cloudinit/net/__init__.py | 37 +- cloudinit/net/bsd.py | 11 +- cloudinit/net/dhcp.py | 7 +- cloudinit/net/eni.py | 9 +- cloudinit/net/freebsd.py | 7 +- cloudinit/net/netbsd.py | 5 +- cloudinit/net/netplan.py | 17 +- cloudinit/net/openbsd.py | 7 +- cloudinit/net/sysconfig.py | 19 +- cloudinit/net/tests/test_dhcp.py | 12 +- cloudinit/net/tests/test_init.py | 5 +- cloudinit/netinfo.py | 27 +- cloudinit/sources/DataSourceAltCloud.py | 8 +- cloudinit/sources/DataSourceAzure.py | 21 +- cloudinit/sources/DataSourceCloudStack.py | 3 +- cloudinit/sources/DataSourceConfigDrive.py | 3 +- cloudinit/sources/DataSourceIBMCloud.py | 3 +- cloudinit/sources/DataSourceOVF.py | 7 +- cloudinit/sources/DataSourceOpenNebula.py | 5 +- cloudinit/sources/DataSourceRbxCloud.py | 7 +- cloudinit/sources/DataSourceSmartOS.py | 5 +- cloudinit/sources/helpers/azure.py | 17 +- cloudinit/sources/helpers/digitalocean.py | 9 +- cloudinit/sources/helpers/openstack.py | 3 +- .../helpers/vmware/imc/config_custom_script.py | 3 +- cloudinit/sources/helpers/vmware/imc/config_nic.py | 7 +- .../sources/helpers/vmware/imc/config_passwd.py | 7 +- .../sources/helpers/vmware/imc/guestcust_util.py | 10 +- cloudinit/subp.py | 301 +++++++++++++++- cloudinit/tests/helpers.py | 13 +- cloudinit/tests/test_conftest.py | 24 +- cloudinit/tests/test_gpg.py | 10 +- cloudinit/tests/test_netinfo.py | 40 +-- cloudinit/tests/test_subp.py | 227 ++++++++++++- cloudinit/tests/test_util.py | 11 +- cloudinit/util.py | 377 ++------------------- packages/bddeb | 9 +- packages/brpm | 5 +- tests/cloud_tests/bddeb.py | 6 +- tests/cloud_tests/platforms/lxd/image.py | 9 +- tests/cloud_tests/platforms/lxd/instance.py | 3 +- tests/cloud_tests/platforms/nocloudkvm/image.py | 12 +- tests/cloud_tests/platforms/nocloudkvm/instance.py | 10 +- tests/cloud_tests/platforms/nocloudkvm/platform.py | 5 +- tests/cloud_tests/platforms/platforms.py | 9 +- tests/cloud_tests/util.py | 7 +- tests/unittests/test_builtin_handlers.py | 3 +- tests/unittests/test_datasource/test_altcloud.py | 7 +- tests/unittests/test_datasource/test_azure.py | 30 +- .../unittests/test_datasource/test_azure_helper.py | 2 +- tests/unittests/test_datasource/test_cloudstack.py | 2 +- tests/unittests/test_datasource/test_ovf.py | 9 +- tests/unittests/test_datasource/test_rbx.py | 10 +- tests/unittests/test_datasource/test_smartos.py | 6 +- tests/unittests/test_distros/test_create_users.py | 8 +- tests/unittests/test_distros/test_debian.py | 2 +- tests/unittests/test_distros/test_freebsd.py | 4 +- tests/unittests/test_distros/test_generic.py | 4 +- tests/unittests/test_distros/test_netconfig.py | 3 +- .../test_distros/test_user_data_normalize.py | 6 +- tests/unittests/test_ds_identify.py | 5 +- .../test_handler_apt_configure_sources_list_v1.py | 5 +- .../test_handler_apt_configure_sources_list_v3.py | 5 +- .../test_handler/test_handler_apt_source_v1.py | 13 +- .../test_handler/test_handler_apt_source_v3.py | 21 +- .../unittests/test_handler/test_handler_bootcmd.py | 6 +- .../test_handler/test_handler_ca_certs.py | 5 +- tests/unittests/test_handler/test_handler_chef.py | 6 +- .../test_handler/test_handler_disk_setup.py | 8 +- .../test_handler/test_handler_growpart.py | 10 +- .../test_handler/test_handler_landscape.py | 6 +- .../unittests/test_handler/test_handler_locale.py | 2 +- tests/unittests/test_handler/test_handler_lxd.py | 28 +- .../test_handler/test_handler_mcollective.py | 7 +- .../unittests/test_handler/test_handler_mounts.py | 6 +- tests/unittests/test_handler/test_handler_ntp.py | 51 +-- .../unittests/test_handler/test_handler_puppet.py | 26 +- .../unittests/test_handler/test_handler_runcmd.py | 4 +- .../test_handler/test_handler_seed_random.py | 5 +- .../test_handler/test_handler_spacewalk.py | 20 +- tests/unittests/test_net.py | 33 +- tests/unittests/test_net_freebsd.py | 2 +- tests/unittests/test_render_cloudcfg.py | 7 +- tests/unittests/test_reporting_hyperv.py | 2 +- tests/unittests/test_rh_subscription.py | 18 +- tests/unittests/test_util.py | 258 ++------------ tests/unittests/test_vmware/test_guestcust_util.py | 22 +- tools/.github-cla-signers | 1 + 148 files changed, 1403 insertions(+), 1292 deletions(-) (limited to 'cloudinit/config/tests') diff --git a/cloudinit/analyze/dump.py b/cloudinit/analyze/dump.py index 939c3126..62ad51fe 100644 --- a/cloudinit/analyze/dump.py +++ b/cloudinit/analyze/dump.py @@ -4,6 +4,7 @@ import calendar from datetime import datetime import sys +from cloudinit import subp from cloudinit import util stage_to_description = { @@ -51,7 +52,7 @@ def parse_timestamp(timestampstr): def parse_timestamp_from_date(timestampstr): - out, _ = util.subp(['date', '+%s.%3N', '-d', timestampstr]) + out, _ = subp.subp(['date', '+%s.%3N', '-d', timestampstr]) timestamp = out.strip() return float(timestamp) diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py index fb152b1d..cca1fa7f 100644 --- a/cloudinit/analyze/show.py +++ b/cloudinit/analyze/show.py @@ -11,6 +11,7 @@ import os import time import sys +from cloudinit import subp from cloudinit import util from cloudinit.distros import uses_systemd @@ -155,7 +156,7 @@ class SystemctlReader(object): :return: whether the subp call failed or not ''' try: - value, err = util.subp(self.args, capture=True) + value, err = subp.subp(self.args, capture=True) if err: return err self.epoch = value @@ -215,7 +216,7 @@ def gather_timestamps_using_dmesg(): with gather_timestamps_using_systemd ''' try: - data, _ = util.subp(['dmesg'], capture=True) + data, _ = subp.subp(['dmesg'], capture=True) split_entries = data[0].splitlines() for i in split_entries: if i.decode('UTF-8').find('user') != -1: diff --git a/cloudinit/analyze/tests/test_boot.py b/cloudinit/analyze/tests/test_boot.py index f4001c14..f69423c3 100644 --- a/cloudinit/analyze/tests/test_boot.py +++ b/cloudinit/analyze/tests/test_boot.py @@ -25,7 +25,7 @@ class TestDistroChecker(CiTestCase): m_get_linux_distro, m_is_FreeBSD): self.assertEqual(err_code, dist_check_timestamp()) - @mock.patch('cloudinit.util.subp', return_value=(0, 1)) + @mock.patch('cloudinit.subp.subp', return_value=(0, 1)) def test_subp_fails(self, m_subp): self.assertEqual(err_code, dist_check_timestamp()) @@ -42,7 +42,7 @@ class TestSystemCtlReader(CiTestCase): with self.assertRaises(RuntimeError): reader.parse_epoch_as_float() - @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None)) def test_systemctl_works_correctly_threshold(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') self.assertEqual(1.0, reader.parse_epoch_as_float()) @@ -50,12 +50,12 @@ class TestSystemCtlReader(CiTestCase): self.assertTrue(thresh < 1e-6) self.assertTrue(thresh > (-1 * 1e-6)) - @mock.patch('cloudinit.util.subp', return_value=('U=0', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=0', None)) def test_systemctl_succeed_zero(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') self.assertEqual(0.0, reader.parse_epoch_as_float()) - @mock.patch('cloudinit.util.subp', return_value=('U=1', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1', None)) def test_systemctl_succeed_distinct(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') val1 = reader.parse_epoch_as_float() @@ -64,13 +64,13 @@ class TestSystemCtlReader(CiTestCase): val2 = reader2.parse_epoch_as_float() self.assertNotEqual(val1, val2) - @mock.patch('cloudinit.util.subp', return_value=('100', None)) + @mock.patch('cloudinit.subp.subp', return_value=('100', None)) def test_systemctl_epoch_not_splittable(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') with self.assertRaises(IndexError): reader.parse_epoch_as_float() - @mock.patch('cloudinit.util.subp', return_value=('U=foobar', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=foobar', None)) def test_systemctl_cannot_convert_epoch_to_float(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') with self.assertRaises(ValueError): @@ -130,7 +130,7 @@ class TestAnalyzeBoot(CiTestCase): self.assertEqual(err_string, data) @mock.patch("cloudinit.util.is_container", return_value=True) - @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None)) def test_container_no_ci_log_line(self, m_is_container, m_subp): path = os.path.dirname(os.path.abspath(__file__)) log_path = path + '/boot-test.log' @@ -148,7 +148,7 @@ class TestAnalyzeBoot(CiTestCase): self.assertEqual(FAIL_CODE, finish_code) @mock.patch("cloudinit.util.is_container", return_value=True) - @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None)) @mock.patch('cloudinit.analyze.__main__._get_events', return_value=[{ 'name': 'init-local', 'description': 'starting search', 'timestamp': 100000}]) diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py index d6fbd381..dac1efb6 100644 --- a/cloudinit/analyze/tests/test_dump.py +++ b/cloudinit/analyze/tests/test_dump.py @@ -5,7 +5,8 @@ from textwrap import dedent from cloudinit.analyze.dump import ( dump_events, parse_ci_logline, parse_timestamp) -from cloudinit.util import which, write_file +from cloudinit.util import write_file +from cloudinit.subp import which from cloudinit.tests.helpers import CiTestCase, mock, skipIf diff --git a/cloudinit/cmd/clean.py b/cloudinit/cmd/clean.py index 30e49de0..928a8eea 100644 --- a/cloudinit/cmd/clean.py +++ b/cloudinit/cmd/clean.py @@ -10,9 +10,8 @@ import os import sys from cloudinit.stages import Init -from cloudinit.util import ( - ProcessExecutionError, del_dir, del_file, get_config_logfiles, - is_link, subp) +from cloudinit.subp import (ProcessExecutionError, subp) +from cloudinit.util import (del_dir, del_file, get_config_logfiles, is_link) def error(msg): diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py index 4c086b51..51c61cca 100644 --- a/cloudinit/cmd/devel/logs.py +++ b/cloudinit/cmd/devel/logs.py @@ -12,8 +12,8 @@ import sys from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE from cloudinit.temp_utils import tempdir -from cloudinit.util import ( - ProcessExecutionError, chdir, copy, ensure_dir, subp, write_file) +from cloudinit.subp import (ProcessExecutionError, subp) +from cloudinit.util import (chdir, copy, ensure_dir, write_file) CLOUDINIT_LOGS = ['/var/log/cloud-init.log', '/var/log/cloud-init-output.log'] diff --git a/cloudinit/cmd/devel/tests/test_logs.py b/cloudinit/cmd/devel/tests/test_logs.py index d2dfa8de..ddfd58e1 100644 --- a/cloudinit/cmd/devel/tests/test_logs.py +++ b/cloudinit/cmd/devel/tests/test_logs.py @@ -8,7 +8,8 @@ from cloudinit.cmd.devel import logs from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE from cloudinit.tests.helpers import ( FilesystemMockingTestCase, mock, wrap_and_call) -from cloudinit.util import ensure_dir, load_file, subp, write_file +from cloudinit.subp import subp +from cloudinit.util import ensure_dir, load_file, write_file @mock.patch('cloudinit.cmd.devel.logs.os.getuid') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index b1c7b471..73d8719f 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -17,6 +17,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit import gpg from cloudinit import log as logging +from cloudinit import subp from cloudinit import templater from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -431,7 +432,7 @@ def _should_configure_on_empty_apt(): # if no config was provided, should apt configuration be done? if util.system_is_snappy(): return False, "system is snappy." - if not (util.which('apt-get') or util.which('apt')): + if not (subp.which('apt-get') or subp.which('apt')): return False, "no apt commands." return True, "Apt is available." @@ -478,7 +479,7 @@ def apply_apt(cfg, cloud, target): def debconf_set_selections(selections, target=None): if not selections.endswith(b'\n'): selections += b'\n' - util.subp(['debconf-set-selections'], data=selections, target=target, + subp.subp(['debconf-set-selections'], data=selections, target=target, capture=True) @@ -503,7 +504,7 @@ def dpkg_reconfigure(packages, target=None): "but cannot be unconfigured: %s", unhandled) if len(to_config): - util.subp(['dpkg-reconfigure', '--frontend=noninteractive'] + + subp.subp(['dpkg-reconfigure', '--frontend=noninteractive'] + list(to_config), data=None, target=target, capture=True) @@ -546,7 +547,7 @@ def apply_debconf_selections(cfg, target=None): def clean_cloud_init(target): """clean out any local cloud-init config""" flist = glob.glob( - util.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) + subp.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) LOG.debug("cleaning cloud-init config from: %s", flist) for dpkg_cfg in flist: @@ -575,7 +576,7 @@ def rename_apt_lists(new_mirrors, target, arch): """rename_apt_lists - rename apt lists to preserve old cache data""" default_mirrors = get_default_mirrors(arch) - pre = util.target_path(target, APT_LISTS) + pre = subp.target_path(target, APT_LISTS) for (name, omirror) in default_mirrors.items(): nmirror = new_mirrors.get(name) if not nmirror: @@ -694,8 +695,8 @@ def add_apt_key_raw(key, target=None): """ LOG.debug("Adding key:\n'%s'", key) try: - util.subp(['apt-key', 'add', '-'], data=key.encode(), target=target) - except util.ProcessExecutionError: + subp.subp(['apt-key', 'add', '-'], data=key.encode(), target=target) + except subp.ProcessExecutionError: LOG.exception("failed to add apt GPG Key to apt keyring") raise @@ -758,13 +759,13 @@ def add_apt_sources(srcdict, cloud, target=None, template_params=None, if aa_repo_match(source): try: - util.subp(["add-apt-repository", source], target=target) - except util.ProcessExecutionError: + subp.subp(["add-apt-repository", source], target=target) + except subp.ProcessExecutionError: LOG.exception("add-apt-repository failed.") raise continue - sourcefn = util.target_path(target, ent['filename']) + sourcefn = subp.target_path(target, ent['filename']) try: contents = "%s\n" % (source) util.write_file(sourcefn, contents, omode="a") diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 6813f534..246e4497 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -16,6 +16,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_ALWAYS from cloudinit import temp_utils +from cloudinit import subp from cloudinit import util frequency = PER_ALWAYS @@ -99,7 +100,7 @@ def handle(name, cfg, cloud, log, _args): if iid: env['INSTANCE_ID'] = str(iid) cmd = ['/bin/sh', tmpf.name] - util.subp(cmd, env=env, capture=False) + subp.subp(cmd, env=env, capture=False) except Exception: util.logexc(log, "Failed to run bootcmd module %s", name) raise diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 0b4352c8..9fdaeba1 100755 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -39,6 +39,7 @@ Valid configuration options for this module are: """ from cloudinit.distros import ug_util +from cloudinit import subp from cloudinit import util distros = ['ubuntu', 'debian'] @@ -93,6 +94,6 @@ def handle(name, cfg, cloud, log, args): if len(shcmd): cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] log.debug("Setting byobu to %s", value) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 64bc900e..7617a8ea 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -36,6 +36,7 @@ can be removed from the system with the configuration option import os +from cloudinit import subp from cloudinit import util CA_CERT_PATH = "/usr/share/ca-certificates/" @@ -51,7 +52,7 @@ def update_ca_certs(): """ Updates the CA certificate cache on the current machine. """ - util.subp(["update-ca-certificates"], capture=False) + subp.subp(["update-ca-certificates"], capture=False) def add_ca_certs(certs): @@ -85,7 +86,7 @@ def remove_default_ca_certs(): util.delete_dir_contents(CA_CERT_SYSTEM_PATH) util.write_file(CA_CERT_CONFIG, "", mode=0o644) debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" - util.subp(('debconf-set-selections', '-'), debconf_sel) + subp.subp(('debconf-set-selections', '-'), debconf_sel) def handle(name, cfg, _cloud, log, _args): diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 03285ef0..e1f51fce 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -76,7 +76,9 @@ import itertools import json import os +from cloudinit import subp from cloudinit import templater +from cloudinit import temp_utils from cloudinit import url_helper from cloudinit import util @@ -282,7 +284,32 @@ def run_chef(chef_cfg, log): cmd.extend(CHEF_EXEC_DEF_ARGS) else: cmd.extend(CHEF_EXEC_DEF_ARGS) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) + + +def subp_blob_in_tempfile(blob, *args, **kwargs): + """Write blob to a tempfile, and call subp with args, kwargs. Then cleanup. + + 'basename' as a kwarg allows providing the basename for the file. + The 'args' argument to subp will be updated with the full path to the + filename as the first argument. + """ + basename = kwargs.pop('basename', "subp_blob") + + if len(args) == 0 and 'args' not in kwargs: + args = [tuple()] + + # Use tmpdir over tmpfile to avoid 'text file busy' on execute + with temp_utils.tempdir(needs_exe=True) as tmpd: + tmpf = os.path.join(tmpd, basename) + if 'args' in kwargs: + kwargs['args'] = [tmpf] + list(kwargs['args']) + else: + args = list(args) + args[0] = [tmpf] + args[0] + + util.write_file(tmpf, blob, mode=0o700) + return subp.subp(*args, **kwargs) def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None): @@ -305,7 +332,7 @@ def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None): else: args = ['-v', omnibus_version] content = url_helper.readurl(url=url, retries=retries).contents - return util.subp_blob_in_tempfile( + return subp_blob_in_tempfile( blob=content, args=args, basename='chef-omnibus-install', capture=False) @@ -354,11 +381,11 @@ def install_chef_from_gems(ruby_version, chef_version, distro): if not os.path.exists('/usr/bin/ruby'): util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') if chef_version: - util.subp(['/usr/bin/gem', 'install', 'chef', + subp.subp(['/usr/bin/gem', 'install', 'chef', '-v %s' % chef_version, '--no-ri', '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) else: - util.subp(['/usr/bin/gem', 'install', 'chef', + subp.subp(['/usr/bin/gem', 'install', 'chef', '--no-ri', '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) diff --git a/cloudinit/config/cc_disable_ec2_metadata.py b/cloudinit/config/cc_disable_ec2_metadata.py index 885b3138..dff93245 100644 --- a/cloudinit/config/cc_disable_ec2_metadata.py +++ b/cloudinit/config/cc_disable_ec2_metadata.py @@ -26,6 +26,7 @@ by default. disable_ec2_metadata: """ +from cloudinit import subp from cloudinit import util from cloudinit.settings import PER_ALWAYS @@ -40,15 +41,15 @@ def handle(name, cfg, _cloud, log, _args): disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False) if disabled: reject_cmd = None - if util.which('ip'): + if subp.which('ip'): reject_cmd = REJECT_CMD_IP - elif util.which('ifconfig'): + elif subp.which('ifconfig'): reject_cmd = REJECT_CMD_IF else: log.error(('Neither "route" nor "ip" command found, unable to ' 'manipulate routing table')) return - util.subp(reject_cmd, capture=False) + subp.subp(reject_cmd, capture=False) else: log.debug(("Skipping module named %s," " disabling the ec2 route not enabled"), name) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 45925755..d957cfe3 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -99,6 +99,7 @@ specified using ``filesystem``. from cloudinit.settings import PER_INSTANCE from cloudinit import util +from cloudinit import subp import logging import os import shlex @@ -106,13 +107,13 @@ import shlex frequency = PER_INSTANCE # Define the commands to use -UDEVADM_CMD = util.which('udevadm') -SFDISK_CMD = util.which("sfdisk") -SGDISK_CMD = util.which("sgdisk") -LSBLK_CMD = util.which("lsblk") -BLKID_CMD = util.which("blkid") -BLKDEV_CMD = util.which("blockdev") -WIPEFS_CMD = util.which("wipefs") +UDEVADM_CMD = subp.which('udevadm') +SFDISK_CMD = subp.which("sfdisk") +SGDISK_CMD = subp.which("sgdisk") +LSBLK_CMD = subp.which("lsblk") +BLKID_CMD = subp.which("blkid") +BLKDEV_CMD = subp.which("blockdev") +WIPEFS_CMD = subp.which("wipefs") LANG_C_ENV = {'LANG': 'C'} @@ -248,7 +249,7 @@ def enumerate_disk(device, nodeps=False): info = None try: - info, _err = util.subp(lsblk_cmd) + info, _err = subp.subp(lsblk_cmd) except Exception as e: raise Exception("Failed during disk check for %s\n%s" % (device, e)) @@ -310,7 +311,7 @@ def check_fs(device): blkid_cmd = [BLKID_CMD, '-c', '/dev/null', device] try: - out, _err = util.subp(blkid_cmd, rcs=[0, 2]) + out, _err = subp.subp(blkid_cmd, rcs=[0, 2]) except Exception as e: raise Exception("Failed during disk check for %s\n%s" % (device, e)) @@ -433,8 +434,8 @@ def get_dyn_func(*args): def get_hdd_size(device): try: - size_in_bytes, _ = util.subp([BLKDEV_CMD, '--getsize64', device]) - sector_size, _ = util.subp([BLKDEV_CMD, '--getss', device]) + size_in_bytes, _ = subp.subp([BLKDEV_CMD, '--getsize64', device]) + sector_size, _ = subp.subp([BLKDEV_CMD, '--getss', device]) except Exception as e: raise Exception("Failed to get %s size\n%s" % (device, e)) @@ -452,7 +453,7 @@ def check_partition_mbr_layout(device, layout): read_parttbl(device) prt_cmd = [SFDISK_CMD, "-l", device] try: - out, _err = util.subp(prt_cmd, data="%s\n" % layout) + out, _err = subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: raise Exception("Error running partition command on %s\n%s" % ( device, e)) @@ -482,7 +483,7 @@ def check_partition_mbr_layout(device, layout): def check_partition_gpt_layout(device, layout): prt_cmd = [SGDISK_CMD, '-p', device] try: - out, _err = util.subp(prt_cmd, update_env=LANG_C_ENV) + out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV) except Exception as e: raise Exception("Error running partition command on %s\n%s" % ( device, e)) @@ -655,7 +656,7 @@ def purge_disk(device): wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']] try: LOG.info("Purging filesystem on /dev/%s", d['name']) - util.subp(wipefs_cmd) + subp.subp(wipefs_cmd) except Exception: raise Exception("Failed FS purge of /dev/%s" % d['name']) @@ -682,7 +683,7 @@ def read_parttbl(device): blkdev_cmd = [BLKDEV_CMD, '--rereadpt', device] util.udevadm_settle() try: - util.subp(blkdev_cmd) + subp.subp(blkdev_cmd) except Exception as e: util.logexc(LOG, "Failed reading the partition table %s" % e) @@ -697,7 +698,7 @@ def exec_mkpart_mbr(device, layout): # Create the partitions prt_cmd = [SFDISK_CMD, "--Linux", "--unit=S", "--force", device] try: - util.subp(prt_cmd, data="%s\n" % layout) + subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: raise Exception("Failed to partition device %s\n%s" % (device, e)) @@ -706,16 +707,16 @@ def exec_mkpart_mbr(device, layout): def exec_mkpart_gpt(device, layout): try: - util.subp([SGDISK_CMD, '-Z', device]) + subp.subp([SGDISK_CMD, '-Z', device]) for index, (partition_type, (start, end)) in enumerate(layout): index += 1 - util.subp([SGDISK_CMD, + subp.subp([SGDISK_CMD, '-n', '{}:{}:{}'.format(index, start, end), device]) if partition_type is not None: # convert to a 4 char (or more) string right padded with 0 # 82 -> 8200. 'Linux' -> 'Linux' pinput = str(partition_type).ljust(4, "0") - util.subp( + subp.subp( [SGDISK_CMD, '-t', '{}:{}'.format(index, pinput), device]) except Exception: LOG.warning("Failed to partition device %s", device) @@ -967,9 +968,9 @@ def mkfs(fs_cfg): fs_cmd) else: # Find the mkfs command - mkfs_cmd = util.which("mkfs.%s" % fs_type) + mkfs_cmd = subp.which("mkfs.%s" % fs_type) if not mkfs_cmd: - mkfs_cmd = util.which("mk%s" % fs_type) + mkfs_cmd = subp.which("mk%s" % fs_type) if not mkfs_cmd: LOG.warning("Cannot create fstype '%s'. No mkfs.%s command", @@ -994,7 +995,7 @@ def mkfs(fs_cfg): LOG.debug("Creating file system %s on %s", label, device) LOG.debug(" Using cmd: %s", str(fs_cmd)) try: - util.subp(fs_cmd, shell=shell) + subp.subp(fs_cmd, shell=shell) except Exception as e: raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index b342e04d..b1d99f97 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -25,7 +25,7 @@ import os from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS -from cloudinit import util +from cloudinit import subp frequency = PER_ALWAYS @@ -43,9 +43,9 @@ def is_upstart_system(): del myenv['UPSTART_SESSION'] check_cmd = ['initctl', 'version'] try: - (out, _err) = util.subp(check_cmd, env=myenv) + (out, _err) = subp.subp(check_cmd, env=myenv) return 'upstart' in out - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: LOG.debug("'%s' returned '%s', not using upstart", ' '.join(check_cmd), e.exit_code) return False @@ -66,7 +66,7 @@ def handle(name, _cfg, cloud, log, args): for n in event_names: cmd = ['initctl', 'emit', str(n), 'CLOUD_CFG=%s' % cfgpath] try: - util.subp(cmd) + subp.subp(cmd) except Exception as e: # TODO(harlowja), use log exception from utils?? log.warning("Emission of upstart event %s failed due to: %s", n, e) diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py index 0a135bbe..77984bca 100644 --- a/cloudinit/config/cc_fan.py +++ b/cloudinit/config/cc_fan.py @@ -39,6 +39,7 @@ If cloud-init sees a ``fan`` entry in cloud-config it will: from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -62,8 +63,8 @@ def stop_update_start(service, config_file, content, systemd=False): def run(cmd, msg): try: - return util.subp(cmd, capture=True) - except util.ProcessExecutionError as e: + return subp.subp(cmd, capture=True) + except subp.ProcessExecutionError as e: LOG.warning("failed: %s (%s): %s", service, cmd, e) return False @@ -94,7 +95,7 @@ def handle(name, cfg, cloud, log, args): util.write_file(mycfg.get('config_path'), mycfg.get('config'), omode="w") distro = cloud.distro - if not util.which('fanctl'): + if not subp.which('fanctl'): distro.install_packages(['ubuntu-fan']) stop_update_start( diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 1b512a06..c5d93f81 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -70,6 +70,7 @@ import stat from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS +from cloudinit import subp from cloudinit import util frequency = PER_ALWAYS @@ -131,19 +132,19 @@ class ResizeGrowPart(object): myenv['LANG'] = 'C' try: - (out, _err) = util.subp(["growpart", "--help"], env=myenv) + (out, _err) = subp.subp(["growpart", "--help"], env=myenv) if re.search(r"--update\s+", out): return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass return False def resize(self, diskdev, partnum, partdev): before = get_size(partdev) try: - util.subp(["growpart", '--dry-run', diskdev, partnum]) - except util.ProcessExecutionError as e: + subp.subp(["growpart", '--dry-run', diskdev, partnum]) + except subp.ProcessExecutionError as e: if e.exit_code != 1: util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)", diskdev, partnum) @@ -151,8 +152,8 @@ class ResizeGrowPart(object): return (before, before) try: - util.subp(["growpart", diskdev, partnum]) - except util.ProcessExecutionError as e: + subp.subp(["growpart", diskdev, partnum]) + except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: growpart %s %s", diskdev, partnum) raise ResizeFailedException(e) @@ -165,11 +166,11 @@ class ResizeGpart(object): myenv['LANG'] = 'C' try: - (_out, err) = util.subp(["gpart", "help"], env=myenv, rcs=[0, 1]) + (_out, err) = subp.subp(["gpart", "help"], env=myenv, rcs=[0, 1]) if re.search(r"gpart recover ", err): return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass return False @@ -182,16 +183,16 @@ class ResizeGpart(object): be recovered. """ try: - util.subp(["gpart", "recover", diskdev]) - except util.ProcessExecutionError as e: + subp.subp(["gpart", "recover", diskdev]) + except subp.ProcessExecutionError as e: if e.exit_code != 0: util.logexc(LOG, "Failed: gpart recover %s", diskdev) raise ResizeFailedException(e) before = get_size(partdev) try: - util.subp(["gpart", "resize", "-i", partnum, diskdev]) - except util.ProcessExecutionError as e: + subp.subp(["gpart", "resize", "-i", partnum, diskdev]) + except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev) raise ResizeFailedException(e) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 7888464e..eb03c664 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -43,8 +43,9 @@ seeded with empty values, and install_devices_empty is set to true. import os +from cloudinit import subp from cloudinit import util -from cloudinit.util import ProcessExecutionError +from cloudinit.subp import ProcessExecutionError distros = ['ubuntu', 'debian'] @@ -59,7 +60,7 @@ def fetch_idevs(log): try: # get the root disk where the /boot directory resides. - disk = util.subp(['grub-probe', '-t', 'disk', '/boot'], + disk = subp.subp(['grub-probe', '-t', 'disk', '/boot'], capture=True)[0].strip() except ProcessExecutionError as e: # grub-common may not be installed, especially on containers @@ -84,7 +85,7 @@ def fetch_idevs(log): try: # check if disk exists and use udevadm to fetch symlinks - devices = util.subp( + devices = subp.subp( ['udevadm', 'info', '--root', '--query=symlink', disk], capture=True )[0].strip().split() @@ -135,7 +136,7 @@ def handle(name, cfg, _cloud, log, _args): (idevs, idevs_empty)) try: - util.subp(['debconf-set-selections'], dconf_sel) + subp.subp(['debconf-set-selections'], dconf_sel) except Exception: util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg") diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index 3d2ded3d..0f2be52b 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -33,6 +33,7 @@ key can be used. By default ``ssh-dss`` keys are not written to console. import os from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util frequency = PER_INSTANCE @@ -64,7 +65,7 @@ def handle(name, cfg, cloud, log, _args): try: cmd = [helper_path, ','.join(fp_blacklist), ','.join(key_blacklist)] - (stdout, _stderr) = util.subp(cmd) + (stdout, _stderr) = subp.subp(cmd) util.multi_log("%s\n" % (stdout.strip()), stderr=False, console=True) except Exception: diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index a9c04d86..299c4d01 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -61,6 +61,7 @@ from io import BytesIO from configobj import ConfigObj from cloudinit import type_utils +from cloudinit import subp from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -116,7 +117,7 @@ def handle(_name, cfg, cloud, log, _args): log.debug("Wrote landscape config file to %s", LSC_CLIENT_CFG_FILE) util.write_file(LS_DEFAULT_FILE, "RUN=1\n") - util.subp(["service", "landscape-client", "restart"]) + subp.subp(["service", "landscape-client", "restart"]) def merge_together(objs): diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 151a9844..7129c9c6 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -48,6 +48,7 @@ lxd-bridge will be configured accordingly. """ from cloudinit import log as logging +from cloudinit import subp from cloudinit import util import os @@ -85,16 +86,16 @@ def handle(name, cfg, cloud, log, args): # Install the needed packages packages = [] - if not util.which("lxd"): + if not subp.which("lxd"): packages.append('lxd') - if init_cfg.get("storage_backend") == "zfs" and not util.which('zfs'): + if init_cfg.get("storage_backend") == "zfs" and not subp.which('zfs'): packages.append('zfsutils-linux') if len(packages): try: cloud.distro.install_packages(packages) - except util.ProcessExecutionError as exc: + except subp.ProcessExecutionError as exc: log.warning("failed to install packages %s: %s", packages, exc) return @@ -104,20 +105,20 @@ def handle(name, cfg, cloud, log, args): 'network_address', 'network_port', 'storage_backend', 'storage_create_device', 'storage_create_loop', 'storage_pool', 'trust_password') - util.subp(['lxd', 'waitready', '--timeout=300']) + subp.subp(['lxd', 'waitready', '--timeout=300']) cmd = ['lxd', 'init', '--auto'] for k in init_keys: if init_cfg.get(k): cmd.extend(["--%s=%s" % (k.replace('_', '-'), str(init_cfg[k]))]) - util.subp(cmd) + subp.subp(cmd) # Set up lxd-bridge if bridge config is given dconf_comm = "debconf-communicate" if bridge_cfg: net_name = bridge_cfg.get("name", _DEFAULT_NETWORK_NAME) if os.path.exists("/etc/default/lxd-bridge") \ - and util.which(dconf_comm): + and subp.which(dconf_comm): # Bridge configured through packaging debconf = bridge_to_debconf(bridge_cfg) @@ -127,7 +128,7 @@ def handle(name, cfg, cloud, log, args): log.debug("Setting lxd debconf via " + dconf_comm) data = "\n".join(["set %s %s" % (k, v) for k, v in debconf.items()]) + "\n" - util.subp(['debconf-communicate'], data) + subp.subp(['debconf-communicate'], data) except Exception: util.logexc(log, "Failed to run '%s' for lxd with" % dconf_comm) @@ -137,7 +138,7 @@ def handle(name, cfg, cloud, log, args): # Run reconfigure log.debug("Running dpkg-reconfigure for lxd") - util.subp(['dpkg-reconfigure', 'lxd', + subp.subp(['dpkg-reconfigure', 'lxd', '--frontend=noninteractive']) else: # Built-in LXD bridge support @@ -264,7 +265,7 @@ def _lxc(cmd): env = {'LC_ALL': 'C', 'HOME': os.environ.get('HOME', '/root'), 'USER': os.environ.get('USER', 'root')} - util.subp(['lxc'] + list(cmd) + ["--force-local"], update_env=env) + subp.subp(['lxc'] + list(cmd) + ["--force-local"], update_env=env) def maybe_cleanup_default(net_name, did_init, create, attach, @@ -286,7 +287,7 @@ def maybe_cleanup_default(net_name, did_init, create, attach, try: _lxc(["network", "delete", net_name]) LOG.debug(msg, net_name, succeeded) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code != 1: raise e LOG.debug(msg, net_name, fail_assume_enoent) @@ -296,7 +297,7 @@ def maybe_cleanup_default(net_name, did_init, create, attach, try: _lxc(["profile", "device", "remove", profile, nic_name]) LOG.debug(msg, nic_name, profile, succeeded) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code != 1: raise e LOG.debug(msg, nic_name, profile, fail_assume_enoent) diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py index 351183f1..41ea4fc9 100644 --- a/cloudinit/config/cc_mcollective.py +++ b/cloudinit/config/cc_mcollective.py @@ -56,6 +56,7 @@ import io from configobj import ConfigObj from cloudinit import log as logging +from cloudinit import subp from cloudinit import util PUBCERT_FILE = "/etc/mcollective/ssl/server-public.pem" @@ -140,6 +141,6 @@ def handle(name, cfg, cloud, log, _args): configure(config=mcollective_cfg['conf']) # restart mcollective to handle updated config - util.subp(['service', 'mcollective', 'restart'], capture=False) + subp.subp(['service', 'mcollective', 'restart'], capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 85a89cd1..e57d1b1f 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -69,6 +69,7 @@ import os.path import re from cloudinit import type_utils +from cloudinit import subp from cloudinit import util # Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0 @@ -252,8 +253,8 @@ def create_swapfile(fname: str, size: str) -> None: 'count=%s' % size] try: - util.subp(cmd, capture=True) - except util.ProcessExecutionError as e: + subp.subp(cmd, capture=True) + except subp.ProcessExecutionError as e: LOG.warning(errmsg, fname, size, method, e) util.del_file(fname) @@ -267,15 +268,15 @@ def create_swapfile(fname: str, size: str) -> None: else: try: create_swap(fname, size, "fallocate") - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: LOG.warning(errmsg, fname, size, "dd", e) LOG.warning("Will attempt with dd.") create_swap(fname, size, "dd") util.chmod(fname, 0o600) try: - util.subp(['mkswap', fname]) - except util.ProcessExecutionError: + subp.subp(['mkswap', fname]) + except subp.ProcessExecutionError: util.del_file(fname) raise @@ -538,9 +539,9 @@ def handle(_name, cfg, cloud, log, _args): for cmd in activate_cmds: fmt = "Activate mounts: %s:" + ' '.join(cmd) try: - util.subp(cmd) + subp.subp(cmd) log.debug(fmt, "PASS") - except util.ProcessExecutionError: + except subp.ProcessExecutionError: log.warning(fmt, "FAIL") util.logexc(log, fmt, "FAIL") diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 3b2c2020..7d3f73ff 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -14,6 +14,7 @@ from cloudinit import log as logging from cloudinit import temp_utils from cloudinit import templater from cloudinit import type_utils +from cloudinit import subp from cloudinit import util from cloudinit.config.schema import get_schema_doc, validate_cloudconfig_schema from cloudinit.settings import PER_INSTANCE @@ -307,7 +308,7 @@ def select_ntp_client(ntp_client, distro): if distro_ntp_client == "auto": for client in distro.preferred_ntp_clients: cfg = distro_cfg.get(client) - if util.which(cfg.get('check_exe')): + if subp.which(cfg.get('check_exe')): LOG.debug('Selected NTP client "%s", already installed', client) clientcfg = cfg @@ -336,7 +337,7 @@ def install_ntp_client(install_func, packages=None, check_exe="ntpd"): @param check_exe: string. The name of a binary that indicates the package the specified package is already installed. """ - if util.which(check_exe): + if subp.which(check_exe): return if packages is None: packages = ['ntp'] @@ -431,7 +432,7 @@ def reload_ntp(service, systemd=False): cmd = ['systemctl', 'reload-or-restart', service] else: cmd = ['service', service, 'restart'] - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def supplemental_schema_validation(ntp_config): @@ -543,7 +544,7 @@ def handle(name, cfg, cloud, log, _args): try: reload_ntp(ntp_client_config['service_name'], systemd=cloud.distro.uses_systemd()) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: LOG.exception("Failed to reload/start ntp service: %s", e) raise diff --git a/cloudinit/config/cc_package_update_upgrade_install.py b/cloudinit/config/cc_package_update_upgrade_install.py index 86afffef..036baf85 100644 --- a/cloudinit/config/cc_package_update_upgrade_install.py +++ b/cloudinit/config/cc_package_update_upgrade_install.py @@ -43,6 +43,7 @@ import os import time from cloudinit import log as logging +from cloudinit import subp from cloudinit import util REBOOT_FILE = "/var/run/reboot-required" @@ -57,7 +58,7 @@ def _multi_cfg_bool_get(cfg, *keys): def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): - util.subp(REBOOT_CMD) + subp.subp(REBOOT_CMD) start = time.time() wait_time = initial_sleep for _i in range(0, wait_attempts): diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 3e81a3c7..41ffb46c 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -56,6 +56,7 @@ import subprocess import time from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util frequency = PER_INSTANCE @@ -71,7 +72,7 @@ def givecmdline(pid): # PID COMM ARGS # 1 init /bin/init -- if util.is_FreeBSD(): - (output, _err) = util.subp(['procstat', '-c', str(pid)]) + (output, _err) = subp.subp(['procstat', '-c', str(pid)]) line = output.splitlines()[1] m = re.search(r'\d+ (\w|\.|-)+\s+(/\w.+)', line) return m.group(2) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index c01f5b8f..635c73bc 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -83,6 +83,7 @@ import yaml from io import StringIO from cloudinit import helpers +from cloudinit import subp from cloudinit import util PUPPET_CONF_PATH = '/etc/puppet/puppet.conf' @@ -105,14 +106,14 @@ class PuppetConstants(object): def _autostart_puppet(log): # Set puppet to automatically start if os.path.exists('/etc/default/puppet'): - util.subp(['sed', '-i', + subp.subp(['sed', '-i', '-e', 's/^START=.*/START=yes/', '/etc/default/puppet'], capture=False) elif os.path.exists('/bin/systemctl'): - util.subp(['/bin/systemctl', 'enable', 'puppet.service'], + subp.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False) elif os.path.exists('/sbin/chkconfig'): - util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) + subp.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) else: log.warning(("Sorry we do not know how to enable" " puppet services on this system")) @@ -203,6 +204,6 @@ def handle(name, cfg, cloud, log, _args): _autostart_puppet(log) # Start puppetd - util.subp(['service', 'puppet', 'start'], capture=False) + subp.subp(['service', 'puppet', 'start'], capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 01dfc125..8de4db30 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -19,6 +19,7 @@ from textwrap import dedent from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_ALWAYS +from cloudinit import subp from cloudinit import util NOBLOCK = "noblock" @@ -88,11 +89,11 @@ def _resize_zfs(mount_point, devpth): def _get_dumpfs_output(mount_point): - return util.subp(['dumpfs', '-m', mount_point])[0] + return subp.subp(['dumpfs', '-m', mount_point])[0] def _get_gpart_output(part): - return util.subp(['gpart', 'show', part])[0] + return subp.subp(['gpart', 'show', part])[0] def _can_skip_resize_ufs(mount_point, devpth): @@ -306,8 +307,8 @@ def handle(name, cfg, _cloud, log, args): def do_resize(resize_cmd, log): try: - util.subp(resize_cmd) - except util.ProcessExecutionError: + subp.subp(resize_cmd) + except subp.ProcessExecutionError: util.logexc(log, "Failed to resize filesystem (cmd=%s)", resize_cmd) raise # TODO(harlowja): Should we add a fsck check after this to make diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 28c79b83..28d62e9d 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -39,6 +39,7 @@ Subscription`` example config. """ from cloudinit import log as logging +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -173,7 +174,7 @@ class SubscriptionManager(object): try: _sub_man_cli(cmd) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: return False return True @@ -200,7 +201,7 @@ class SubscriptionManager(object): try: return_out = _sub_man_cli(cmd, logstring_val=True)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.stdout == "": self.log_warn("Registration failed due " "to: {0}".format(e.stderr)) @@ -223,7 +224,7 @@ class SubscriptionManager(object): # Attempting to register the system only try: return_out = _sub_man_cli(cmd, logstring_val=True)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.stdout == "": self.log_warn("Registration failed due " "to: {0}".format(e.stderr)) @@ -246,7 +247,7 @@ class SubscriptionManager(object): try: return_out = _sub_man_cli(cmd)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.stdout.rstrip() != '': for line in e.stdout.split("\n"): if line != '': @@ -264,7 +265,7 @@ class SubscriptionManager(object): cmd = ['attach', '--auto'] try: return_out = _sub_man_cli(cmd)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: self.log_warn("Auto-attach failed with: {0}".format(e)) return False for line in return_out.split("\n"): @@ -341,7 +342,7 @@ class SubscriptionManager(object): "system: %s", (", ".join(pool_list)) .replace('--pool=', '')) return True - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: self.log_warn("Unable to attach pool {0} " "due to {1}".format(pool, e)) return False @@ -414,7 +415,7 @@ class SubscriptionManager(object): try: _sub_man_cli(cmd) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: self.log_warn("Unable to alter repos due to {0}".format(e)) return False @@ -432,11 +433,11 @@ class SubscriptionManager(object): def _sub_man_cli(cmd, logstring_val=False): ''' - Uses the prefered cloud-init subprocess def of util.subp + Uses the prefered cloud-init subprocess def of subp.subp and runs subscription-manager. Breaking this to a separate function for later use in mocking and unittests ''' - return util.subp(['subscription-manager'] + cmd, + return subp.subp(['subscription-manager'] + cmd, logstring=logstring_val) diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 5df0137d..1354885a 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -182,6 +182,7 @@ import os import re from cloudinit import log as logging +from cloudinit import subp from cloudinit import util DEF_FILENAME = "20-cloud-config.conf" @@ -215,7 +216,7 @@ def reload_syslog(command=DEF_RELOAD, systemd=False): cmd = ['service', service, 'restart'] else: cmd = command - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def load_config(cfg): @@ -429,7 +430,7 @@ def handle(name, cfg, cloud, log, _args): restarted = reload_syslog( command=mycfg[KEYNAME_RELOAD], systemd=cloud.distro.uses_systemd()), - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: restarted = False log.warning("Failed to reload syslog", e) diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 5dd8de37..b61876aa 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -45,7 +45,7 @@ specify them with ``pkg_name``, ``service_name`` and ``config_dir``. import os -from cloudinit import safeyaml, util +from cloudinit import safeyaml, subp, util from cloudinit.distros import rhel_util @@ -130,6 +130,6 @@ def handle(name, cfg, cloud, log, _args): # restart salt-minion. 'service' will start even if not started. if it # was started, it needs to be restarted for config change. - util.subp(['service', const.srv_name, 'restart'], capture=False) + subp.subp(['service', const.srv_name, 'restart'], capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index b65f3ed9..4fb9b44e 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -65,6 +65,7 @@ from io import BytesIO from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util frequency = PER_INSTANCE @@ -92,14 +93,14 @@ def handle_random_seed_command(command, required, env=None): return cmd = command[0] - if not util.which(cmd): + if not subp.which(cmd): if required: raise ValueError( "command '{cmd}' not found but required=true".format(cmd=cmd)) else: LOG.debug("command '%s' not found for seed_command", cmd) return - util.subp(command, env=env, capture=False) + subp.subp(command, env=env, capture=False) def handle(name, cfg, cloud, log, _args): diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 7b7aa885..d6b5682d 100755 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -83,6 +83,7 @@ import sys from cloudinit.distros import ug_util from cloudinit import log as logging from cloudinit.ssh_util import update_ssh_config +from cloudinit import subp from cloudinit import util from string import ascii_letters, digits @@ -128,7 +129,7 @@ def handle_ssh_pwauth(pw_auth, service_cmd=None, service_name="ssh"): cmd = list(service_cmd) + ["restart", service_name] else: cmd = list(service_cmd) + [service_name, "restart"] - util.subp(cmd) + subp.subp(cmd) LOG.debug("Restarted the SSH daemon.") @@ -247,6 +248,6 @@ def chpasswd(distro, plist_in, hashed=False): distro.set_passwd(u, p, hashed=hashed) else: cmd = ['chpasswd'] + (['-e'] if hashed else []) - util.subp(cmd, plist_in) + subp.subp(cmd, plist_in) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py index 8178562e..20ed7d2f 100644 --- a/cloudinit/config/cc_snap.py +++ b/cloudinit/config/cc_snap.py @@ -12,6 +12,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_INSTANCE from cloudinit.subp import prepend_base_command +from cloudinit import subp from cloudinit import util @@ -175,7 +176,7 @@ def add_assertions(assertions): LOG.debug('Snap acking: %s', asrt.split('\n')[0:2]) util.write_file(ASSERTIONS_FILE, combined.encode('utf-8')) - util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) + subp.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) def run_commands(commands): @@ -204,8 +205,8 @@ def run_commands(commands): for command in fixed_snap_commands: shell = isinstance(command, str) try: - util.subp(command, shell=shell, status_cb=sys.stderr.write) - except util.ProcessExecutionError as e: + subp.subp(command, shell=shell, status_cb=sys.stderr.write) + except subp.ProcessExecutionError as e: cmd_failures.append(str(e)) if cmd_failures: msg = 'Failures running snap commands:\n{cmd_failures}'.format( diff --git a/cloudinit/config/cc_spacewalk.py b/cloudinit/config/cc_spacewalk.py index 1020e944..95083607 100644 --- a/cloudinit/config/cc_spacewalk.py +++ b/cloudinit/config/cc_spacewalk.py @@ -27,7 +27,7 @@ For more information about spacewalk see: https://fedorahosted.org/spacewalk/ activation_key: """ -from cloudinit import util +from cloudinit import subp distros = ['redhat', 'fedora'] @@ -41,9 +41,9 @@ def is_registered(): # assume we aren't registered; which is sorta ghetto... already_registered = False try: - util.subp(['rhn-profile-sync', '--verbose'], capture=False) + subp.subp(['rhn-profile-sync', '--verbose'], capture=False) already_registered = True - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code != 1: raise return already_registered @@ -65,7 +65,7 @@ def do_register(server, profile_name, cmd.extend(['--sslCACert', str(ca_cert_path)]) if activation_key: cmd.extend(['--activationkey', str(activation_key)]) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def handle(name, cfg, cloud, log, _args): diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 163cce99..228e5e0d 100755 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -116,6 +116,7 @@ import sys from cloudinit.distros import ug_util from cloudinit import ssh_util +from cloudinit import subp from cloudinit import util @@ -164,7 +165,7 @@ def handle(_name, cfg, cloud, log, _args): try: # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) except Exception: util.logexc(log, "Failed generated a key for %s from %s", @@ -186,9 +187,9 @@ def handle(_name, cfg, cloud, log, _args): # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): try: - out, err = util.subp(cmd, capture=True, env=lang_c) + out, err = subp.subp(cmd, capture=True, env=lang_c) sys.stdout.write(util.decode_binary(out)) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: err = util.decode_binary(e.stderr).lower() if (e.exit_code == 1 and err.lower().startswith("unknown key")): diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 63f87298..856e5a9e 100755 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -31,6 +31,7 @@ either ``lp:`` for launchpad or ``gh:`` for github to the username. """ from cloudinit.distros import ug_util +from cloudinit import subp from cloudinit import util import pwd @@ -101,8 +102,8 @@ def import_ssh_ids(ids, user, log): log.debug("Importing SSH ids for user %s.", user) try: - util.subp(cmd, capture=False) - except util.ProcessExecutionError as exc: + subp.subp(cmd, capture=False) + except subp.ProcessExecutionError as exc: util.logexc(log, "Failed to run command to import %s SSH ids", user) raise exc diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py index 8b6d2a1a..35ded5db 100644 --- a/cloudinit/config/cc_ubuntu_advantage.py +++ b/cloudinit/config/cc_ubuntu_advantage.py @@ -8,6 +8,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util @@ -109,8 +110,8 @@ def configure_ua(token=None, enable=None): attach_cmd = ['ua', 'attach', token] LOG.debug('Attaching to Ubuntu Advantage. %s', ' '.join(attach_cmd)) try: - util.subp(attach_cmd) - except util.ProcessExecutionError as e: + subp.subp(attach_cmd) + except subp.ProcessExecutionError as e: msg = 'Failure attaching Ubuntu Advantage:\n{error}'.format( error=str(e)) util.logexc(LOG, msg) @@ -119,8 +120,8 @@ def configure_ua(token=None, enable=None): for service in enable: try: cmd = ['ua', 'enable', service] - util.subp(cmd, capture=True) - except util.ProcessExecutionError as e: + subp.subp(cmd, capture=True) + except subp.ProcessExecutionError as e: enable_errors.append((service, e)) if enable_errors: for service, error in enable_errors: @@ -135,7 +136,7 @@ def configure_ua(token=None, enable=None): def maybe_install_ua_tools(cloud): """Install ubuntu-advantage-tools if not present.""" - if util.which('ua'): + if subp.which('ua'): return try: cloud.distro.update_package_sources() diff --git a/cloudinit/config/cc_ubuntu_drivers.py b/cloudinit/config/cc_ubuntu_drivers.py index 297451d6..2d1d2b32 100644 --- a/cloudinit/config/cc_ubuntu_drivers.py +++ b/cloudinit/config/cc_ubuntu_drivers.py @@ -9,6 +9,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import temp_utils from cloudinit import type_utils from cloudinit import util @@ -108,7 +109,7 @@ def install_drivers(cfg, pkg_install_func): LOG.debug("Not installing NVIDIA drivers. %s=%s", cfgpath, nv_acc) return - if not util.which('ubuntu-drivers'): + if not subp.which('ubuntu-drivers'): LOG.debug("'ubuntu-drivers' command not available. " "Installing ubuntu-drivers-common") pkg_install_func(['ubuntu-drivers-common']) @@ -131,7 +132,7 @@ def install_drivers(cfg, pkg_install_func): debconf_script, util.encode_text(NVIDIA_DRIVER_LATELINK_DEBCONF_SCRIPT), mode=0o755) - util.subp([debconf_script, debconf_file]) + subp.subp([debconf_script, debconf_file]) except Exception as e: util.logexc( LOG, "Failed to register NVIDIA debconf template: %s", str(e)) @@ -141,8 +142,8 @@ def install_drivers(cfg, pkg_install_func): util.del_dir(tdir) try: - util.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg]) - except util.ProcessExecutionError as exc: + subp.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg]) + except subp.ProcessExecutionError as exc: if OLD_UBUNTU_DRIVERS_STDERR_NEEDLE in exc.stderr: LOG.warning('the available version of ubuntu-drivers is' ' too old to perform requested driver installation') diff --git a/cloudinit/config/tests/test_disable_ec2_metadata.py b/cloudinit/config/tests/test_disable_ec2_metadata.py index 823917c7..b00f2083 100644 --- a/cloudinit/config/tests/test_disable_ec2_metadata.py +++ b/cloudinit/config/tests/test_disable_ec2_metadata.py @@ -15,8 +15,8 @@ DISABLE_CFG = {'disable_ec2_metadata': 'true'} class TestEC2MetadataRoute(CiTestCase): - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp') def test_disable_ifconfig(self, m_subp, m_which): """Set the route if ifconfig command is available""" m_which.side_effect = lambda x: x if x == 'ifconfig' else None @@ -25,8 +25,8 @@ class TestEC2MetadataRoute(CiTestCase): ['route', 'add', '-host', '169.254.169.254', 'reject'], capture=False) - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp') def test_disable_ip(self, m_subp, m_which): """Set the route if ip command is available""" m_which.side_effect = lambda x: x if x == 'ip' else None @@ -35,8 +35,8 @@ class TestEC2MetadataRoute(CiTestCase): ['ip', 'route', 'add', 'prohibit', '169.254.169.254'], capture=False) - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp') def test_disable_no_tool(self, m_subp, m_which): """Log error when neither route nor ip commands are available""" m_which.return_value = None # Find neither ifconfig nor ip diff --git a/cloudinit/config/tests/test_grub_dpkg.py b/cloudinit/config/tests/test_grub_dpkg.py index 01efa330..99c05bb5 100644 --- a/cloudinit/config/tests/test_grub_dpkg.py +++ b/cloudinit/config/tests/test_grub_dpkg.py @@ -4,7 +4,7 @@ import pytest from unittest import mock from logging import Logger -from cloudinit.util import ProcessExecutionError +from cloudinit.subp import ProcessExecutionError from cloudinit.config.cc_grub_dpkg import fetch_idevs, handle @@ -79,7 +79,7 @@ class TestFetchIdevs: ) @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc") @mock.patch("cloudinit.config.cc_grub_dpkg.os.path.exists") - @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp") + @mock.patch("cloudinit.config.cc_grub_dpkg.subp.subp") def test_fetch_idevs(self, m_subp, m_exists, m_logexc, grub_output, path_exists, expected_log_call, udevadm_output, expected_idevs): @@ -158,7 +158,7 @@ class TestHandle: @mock.patch("cloudinit.config.cc_grub_dpkg.fetch_idevs") @mock.patch("cloudinit.config.cc_grub_dpkg.util.get_cfg_option_str") @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc") - @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp") + @mock.patch("cloudinit.config.cc_grub_dpkg.subp.subp") def test_handle(self, m_subp, m_logexc, m_get_cfg_str, m_fetch_idevs, cfg_idevs, cfg_idevs_empty, fetch_idevs_output, expected_log_output): diff --git a/cloudinit/config/tests/test_mounts.py b/cloudinit/config/tests/test_mounts.py index 80b54d0f..764a33e3 100644 --- a/cloudinit/config/tests/test_mounts.py +++ b/cloudinit/config/tests/test_mounts.py @@ -13,12 +13,12 @@ class TestCreateSwapfile: @pytest.mark.parametrize('fstype', ('xfs', 'btrfs', 'ext4', 'other')) @mock.patch(M_PATH + 'util.get_mount_info') - @mock.patch(M_PATH + 'util.subp') + @mock.patch(M_PATH + 'subp.subp') def test_happy_path(self, m_subp, m_get_mount_info, fstype, tmpdir): swap_file = tmpdir.join("swap-file") fname = str(swap_file) - # Some of the calls to util.subp should create the swap file; this + # Some of the calls to subp.subp should create the swap file; this # roughly approximates that m_subp.side_effect = lambda *args, **kwargs: swap_file.write('') diff --git a/cloudinit/config/tests/test_set_passwords.py b/cloudinit/config/tests/test_set_passwords.py index 2732bd60..daa1ef51 100644 --- a/cloudinit/config/tests/test_set_passwords.py +++ b/cloudinit/config/tests/test_set_passwords.py @@ -14,7 +14,7 @@ class TestHandleSshPwauth(CiTestCase): with_logs = True - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_unknown_value_logs_warning(self, m_subp): setpass.handle_ssh_pwauth("floo") self.assertIn("Unrecognized value: ssh_pwauth=floo", @@ -22,7 +22,7 @@ class TestHandleSshPwauth(CiTestCase): m_subp.assert_not_called() @mock.patch(MODPATH + "update_ssh_config", return_value=True) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_systemctl_as_service_cmd(self, m_subp, m_update_ssh_config): """If systemctl in service cmd: systemctl restart name.""" setpass.handle_ssh_pwauth( @@ -31,7 +31,7 @@ class TestHandleSshPwauth(CiTestCase): m_subp.call_args) @mock.patch(MODPATH + "update_ssh_config", return_value=True) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_service_as_service_cmd(self, m_subp, m_update_ssh_config): """If systemctl in service cmd: systemctl restart name.""" setpass.handle_ssh_pwauth( @@ -40,7 +40,7 @@ class TestHandleSshPwauth(CiTestCase): m_subp.call_args) @mock.patch(MODPATH + "update_ssh_config", return_value=False) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_not_restarted_if_not_updated(self, m_subp, m_update_ssh_config): """If config is not updated, then no system restart should be done.""" setpass.handle_ssh_pwauth(True) @@ -48,7 +48,7 @@ class TestHandleSshPwauth(CiTestCase): self.assertIn("No need to restart SSH", self.logs.getvalue()) @mock.patch(MODPATH + "update_ssh_config", return_value=True) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_unchanged_does_nothing(self, m_subp, m_update_ssh_config): """If 'unchanged', then no updates to config and no restart.""" setpass.handle_ssh_pwauth( @@ -56,7 +56,7 @@ class TestHandleSshPwauth(CiTestCase): m_update_ssh_config.assert_not_called() m_subp.assert_not_called() - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_valid_change_values(self, m_subp): """If value is a valid changen value, then update should be called.""" upname = MODPATH + "update_ssh_config" @@ -88,7 +88,7 @@ class TestSetPasswordsHandle(CiTestCase): 'ssh_pwauth=None\n', self.logs.getvalue()) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_handle_on_chpasswd_list_parses_common_hashes(self, m_subp): """handle parses command password hashes.""" cloud = self.tmp_cloud(distro='ubuntu') @@ -98,7 +98,7 @@ class TestSetPasswordsHandle(CiTestCase): 'ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoakMMC7dR52q' 'SDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXazGGx3oo1'] cfg = {'chpasswd': {'list': valid_hashed_pwds}} - with mock.patch(MODPATH + 'util.subp') as m_subp: + with mock.patch(MODPATH + 'subp.subp') as m_subp: setpass.handle( 'IGNORED', cfg=cfg, cloud=cloud, log=self.logger, args=[]) self.assertIn( @@ -113,7 +113,7 @@ class TestSetPasswordsHandle(CiTestCase): m_subp.call_args_list) @mock.patch(MODPATH + "util.is_BSD") - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_bsd_calls_custom_pw_cmds_to_set_and_expire_passwords( self, m_subp, m_is_bsd): """BSD don't use chpasswd""" @@ -130,7 +130,7 @@ class TestSetPasswordsHandle(CiTestCase): m_subp.call_args_list) @mock.patch(MODPATH + "util.is_BSD") - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_handle_on_chpasswd_list_creates_random_passwords(self, m_subp, m_is_bsd): """handle parses command set random passwords.""" @@ -140,7 +140,7 @@ class TestSetPasswordsHandle(CiTestCase): 'root:R', 'ubuntu:RANDOM'] cfg = {'chpasswd': {'expire': 'false', 'list': valid_random_pwds}} - with mock.patch(MODPATH + 'util.subp') as m_subp: + with mock.patch(MODPATH + 'subp.subp') as m_subp: setpass.handle( 'IGNORED', cfg=cfg, cloud=cloud, log=self.logger, args=[]) self.assertIn( diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py index 95270fa0..6d4c014a 100644 --- a/cloudinit/config/tests/test_snap.py +++ b/cloudinit/config/tests/test_snap.py @@ -92,7 +92,7 @@ class TestAddAssertions(CiTestCase): super(TestAddAssertions, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_add_assertions_on_empty_list(self, m_subp): """When provided with an empty list, add_assertions does nothing.""" add_assertions([]) @@ -107,7 +107,7 @@ class TestAddAssertions(CiTestCase): "assertion parameter was not a list or dict: I'm Not Valid", str(context_manager.exception)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_add_assertions_adds_assertions_as_list(self, m_subp): """When provided with a list, add_assertions adds all assertions.""" self.assertEqual( @@ -130,7 +130,7 @@ class TestAddAssertions(CiTestCase): self.assertEqual( util.load_file(compare_file), util.load_file(assert_file)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_add_assertions_adds_assertions_as_dict(self, m_subp): """When provided with a dict, add_assertions adds all assertions.""" self.assertEqual( @@ -168,7 +168,7 @@ class TestRunCommands(CiTestCase): super(TestRunCommands, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_run_commands_on_empty_list(self, m_subp): """When provided with an empty list, run_commands does nothing.""" run_commands([]) @@ -477,7 +477,7 @@ class TestHandle(CiTestCase): self.assertEqual('HI\nMOM\n', util.load_file(outfile)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_handle_adds_assertions(self, m_subp): """Any configured snap assertions are provided to add_assertions.""" assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) @@ -493,7 +493,7 @@ class TestHandle(CiTestCase): self.assertEqual( util.load_file(compare_file), util.load_file(assert_file)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') @skipUnlessJsonSchema() def test_handle_validates_schema(self, m_subp): """Any provided configuration is runs validate_cloudconfig_schema.""" diff --git a/cloudinit/config/tests/test_ubuntu_advantage.py b/cloudinit/config/tests/test_ubuntu_advantage.py index 8c4161ef..db7fb726 100644 --- a/cloudinit/config/tests/test_ubuntu_advantage.py +++ b/cloudinit/config/tests/test_ubuntu_advantage.py @@ -3,7 +3,7 @@ from cloudinit.config.cc_ubuntu_advantage import ( configure_ua, handle, maybe_install_ua_tools, schema) from cloudinit.config.schema import validate_cloudconfig_schema -from cloudinit import util +from cloudinit import subp from cloudinit.tests.helpers import ( CiTestCase, mock, SchemaTestCaseMixin, skipUnlessJsonSchema) @@ -26,10 +26,10 @@ class TestConfigureUA(CiTestCase): super(TestConfigureUA, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_error(self, m_subp): """Errors from ua attach command are raised.""" - m_subp.side_effect = util.ProcessExecutionError( + m_subp.side_effect = subp.ProcessExecutionError( 'Invalid token SomeToken') with self.assertRaises(RuntimeError) as context_manager: configure_ua(token='SomeToken') @@ -39,7 +39,7 @@ class TestConfigureUA(CiTestCase): 'Stdout: Invalid token SomeToken\nStderr: -', str(context_manager.exception)) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_token(self, m_subp): """When token is provided, attach the machine to ua using the token.""" configure_ua(token='SomeToken') @@ -48,7 +48,7 @@ class TestConfigureUA(CiTestCase): 'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n', self.logs.getvalue()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_on_service_error(self, m_subp): """all services should be enabled and then any failures raised""" @@ -56,7 +56,7 @@ class TestConfigureUA(CiTestCase): fail_cmds = [['ua', 'enable', svc] for svc in ['esm', 'cc']] if cmd in fail_cmds and capture: svc = cmd[-1] - raise util.ProcessExecutionError( + raise subp.ProcessExecutionError( 'Invalid {} credentials'.format(svc.upper())) m_subp.side_effect = fake_subp @@ -83,7 +83,7 @@ class TestConfigureUA(CiTestCase): 'Failure enabling Ubuntu Advantage service(s): "esm", "cc"', str(context_manager.exception)) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_empty_services(self, m_subp): """When services is an empty list, do not auto-enable attach.""" configure_ua(token='SomeToken', enable=[]) @@ -92,7 +92,7 @@ class TestConfigureUA(CiTestCase): 'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n', self.logs.getvalue()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_specific_services(self, m_subp): """When services a list, only enable specific services.""" configure_ua(token='SomeToken', enable=['fips']) @@ -105,7 +105,7 @@ class TestConfigureUA(CiTestCase): self.logs.getvalue()) @mock.patch('%s.maybe_install_ua_tools' % MPATH, mock.MagicMock()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_string_services(self, m_subp): """When services a string, treat as singleton list and warn""" configure_ua(token='SomeToken', enable='fips') @@ -119,7 +119,7 @@ class TestConfigureUA(CiTestCase): 'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n', self.logs.getvalue()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_weird_services(self, m_subp): """When services not string or list, warn but still attach""" configure_ua(token='SomeToken', enable={'deffo': 'wont work'}) @@ -285,7 +285,7 @@ class TestMaybeInstallUATools(CiTestCase): super(TestMaybeInstallUATools, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_tools_noop_when_ua_tools_present(self, m_which): """Do nothing if ubuntu-advantage-tools already exists.""" m_which.return_value = '/usr/bin/ua' # already installed @@ -294,7 +294,7 @@ class TestMaybeInstallUATools(CiTestCase): 'Some apt error') maybe_install_ua_tools(cloud=FakeCloud(distro)) # No RuntimeError - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_tools_raises_update_errors(self, m_which): """maybe_install_ua_tools logs and raises apt update errors.""" m_which.return_value = None @@ -306,7 +306,7 @@ class TestMaybeInstallUATools(CiTestCase): self.assertEqual('Some apt error', str(context_manager.exception)) self.assertIn('Package update failed\nTraceback', self.logs.getvalue()) - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_raises_install_errors(self, m_which): """maybe_install_ua_tools logs and raises package install errors.""" m_which.return_value = None @@ -320,7 +320,7 @@ class TestMaybeInstallUATools(CiTestCase): self.assertIn( 'Failed to install ubuntu-advantage-tools\n', self.logs.getvalue()) - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_tools_happy_path(self, m_which): """maybe_install_ua_tools installs ubuntu-advantage-tools.""" m_which.return_value = None diff --git a/cloudinit/config/tests/test_ubuntu_drivers.py b/cloudinit/config/tests/test_ubuntu_drivers.py index 0aec1265..504ba356 100644 --- a/cloudinit/config/tests/test_ubuntu_drivers.py +++ b/cloudinit/config/tests/test_ubuntu_drivers.py @@ -7,7 +7,7 @@ from cloudinit.tests.helpers import CiTestCase, skipUnlessJsonSchema, mock from cloudinit.config.schema import ( SchemaValidationError, validate_cloudconfig_schema) from cloudinit.config import cc_ubuntu_drivers as drivers -from cloudinit.util import ProcessExecutionError +from cloudinit.subp import ProcessExecutionError MPATH = "cloudinit.config.cc_ubuntu_drivers." M_TMP_PATH = MPATH + "temp_utils.mkdtemp" @@ -53,8 +53,8 @@ class TestUbuntuDrivers(CiTestCase): schema=drivers.schema, strict=True) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=False) def _assert_happy_path_taken( self, config, m_which, m_subp, m_tmp): """Positive path test through handle. Package should be installed.""" @@ -80,8 +80,8 @@ class TestUbuntuDrivers(CiTestCase): self._assert_happy_path_taken(new_config) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp") - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp") + @mock.patch(MPATH + "subp.which", return_value=False) def test_handle_raises_error_if_no_drivers_found( self, m_which, m_subp, m_tmp): """If ubuntu-drivers doesn't install any drivers, raise an error.""" @@ -109,8 +109,8 @@ class TestUbuntuDrivers(CiTestCase): self.assertIn('ubuntu-drivers found no drivers for installation', self.logs.getvalue()) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=False) def _assert_inert_with_config(self, config, m_which, m_subp): """Helper to reduce repetition when testing negative cases""" myCloud = mock.MagicMock() @@ -154,8 +154,8 @@ class TestUbuntuDrivers(CiTestCase): self.assertEqual(0, m_install_drivers.call_count) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=True) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=True) def test_install_drivers_no_install_if_present( self, m_which, m_subp, m_tmp): """If 'ubuntu-drivers' is present, no package install should occur.""" @@ -181,8 +181,8 @@ class TestUbuntuDrivers(CiTestCase): self.assertEqual(0, pkg_install.call_count) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp") - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp") + @mock.patch(MPATH + "subp.which", return_value=False) def test_install_drivers_handles_old_ubuntu_drivers_gracefully( self, m_which, m_subp, m_tmp): """Older ubuntu-drivers versions should emit message and raise error""" @@ -219,8 +219,8 @@ class TestUbuntuDriversWithVersion(TestUbuntuDrivers): install_gpgpu = ['ubuntu-drivers', 'install', '--gpgpu', 'nvidia:123'] @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=False) def test_version_none_uses_latest(self, m_which, m_subp, m_tmp): tdir = self.tmp_dir() debconf_file = os.path.join(tdir, 'nvidia.template') diff --git a/cloudinit/conftest.py b/cloudinit/conftest.py index 37cbbcda..251bca59 100644 --- a/cloudinit/conftest.py +++ b/cloudinit/conftest.py @@ -2,32 +2,32 @@ from unittest import mock import pytest -from cloudinit import util +from cloudinit import subp @pytest.yield_fixture(autouse=True) def disable_subp_usage(request): """ - Across all (pytest) tests, ensure that util.subp is not invoked. + Across all (pytest) tests, ensure that subp.subp is not invoked. Note that this can only catch invocations where the util module is imported - and ``util.subp(...)`` is called. ``from cloudinit.util import subp`` + and ``subp.subp(...)`` is called. ``from cloudinit.subp mport subp`` imports happen before the patching here (or the CiTestCase monkey-patching) happens, so are left untouched. - To allow a particular test method or class to use util.subp you can set the + To allow a particular test method or class to use subp.subp you can set the parameter passed to this fixture to False using pytest.mark.parametrize:: @pytest.mark.parametrize("disable_subp_usage", [False], indirect=True) def test_whoami(self): - util.subp(["whoami"]) + subp.subp(["whoami"]) - To instead allow util.subp usage for a specific command, you can set the + To instead allow subp.subp usage for a specific command, you can set the parameter passed to this fixture to that command: @pytest.mark.parametrize("disable_subp_usage", ["bash"], indirect=True) def test_bash(self): - util.subp(["bash"]) + subp.subp(["bash"]) To specify multiple commands, set the parameter to a list (note the double-layered list: we specify a single parameter that is itself a list): @@ -35,8 +35,8 @@ def disable_subp_usage(request): @pytest.mark.parametrize( "disable_subp_usage", ["bash", "whoami"], indirect=True) def test_several_things(self): - util.subp(["bash"]) - util.subp(["whoami"]) + subp.subp(["bash"]) + subp.subp(["whoami"]) This fixture (roughly) mirrors the functionality of CiTestCase.allowed_subp. N.B. While autouse fixtures do affect non-pytest @@ -47,11 +47,11 @@ def disable_subp_usage(request): if should_disable: if not isinstance(should_disable, (list, str)): def side_effect(args, *other_args, **kwargs): - raise AssertionError("Unexpectedly used util.subp") + raise AssertionError("Unexpectedly used subp.subp") else: # Look this up before our patch is in place, so we have access to # the real implementation in side_effect - subp = util.subp + real_subp = subp.subp if isinstance(should_disable, str): should_disable = [should_disable] @@ -60,12 +60,12 @@ def disable_subp_usage(request): cmd = args[0] if cmd not in should_disable: raise AssertionError( - "Unexpectedly used util.subp to call {} (allowed:" + "Unexpectedly used subp.subp to call {} (allowed:" " {})".format(cmd, ",".join(should_disable)) ) - return subp(args, *other_args, **kwargs) + return real_subp(args, *other_args, **kwargs) - with mock.patch('cloudinit.util.subp', autospec=True) as m_subp: + with mock.patch('cloudinit.subp.subp', autospec=True) as m_subp: m_subp.side_effect = side_effect yield else: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 35a10590..016ba64d 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -25,6 +25,7 @@ from cloudinit.net import network_state from cloudinit.net import renderers from cloudinit import ssh_util from cloudinit import type_utils +from cloudinit import subp from cloudinit import util from cloudinit.distros.parsers import hosts @@ -225,8 +226,8 @@ class Distro(metaclass=abc.ABCMeta): LOG.debug("Non-persistently setting the system hostname to %s", hostname) try: - util.subp(['hostname', hostname]) - except util.ProcessExecutionError: + subp.subp(['hostname', hostname]) + except subp.ProcessExecutionError: util.logexc(LOG, "Failed to non-persistently adjust the system " "hostname to %s", hostname) @@ -361,12 +362,12 @@ class Distro(metaclass=abc.ABCMeta): LOG.debug("Attempting to run bring up interface %s using command %s", device_name, cmd) try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False @@ -480,7 +481,7 @@ class Distro(metaclass=abc.ABCMeta): # Run the command LOG.debug("Adding user %s", name) try: - util.subp(useradd_cmd, logstring=log_useradd_cmd) + subp.subp(useradd_cmd, logstring=log_useradd_cmd) except Exception as e: util.logexc(LOG, "Failed to create user %s", name) raise e @@ -500,7 +501,7 @@ class Distro(metaclass=abc.ABCMeta): # Run the command LOG.debug("Adding snap user %s", name) try: - (out, err) = util.subp(create_user_cmd, logstring=create_user_cmd, + (out, err) = subp.subp(create_user_cmd, logstring=create_user_cmd, capture=True) LOG.debug("snap create-user returned: %s:%s", out, err) jobj = util.load_json(out) @@ -582,20 +583,20 @@ class Distro(metaclass=abc.ABCMeta): # passwd must use short '-l' due to SLES11 lacking long form '--lock' lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name]) try: - cmd = next(tool for tool in lock_tools if util.which(tool[0])) + cmd = next(tool for tool in lock_tools if subp.which(tool[0])) except StopIteration: raise RuntimeError(( "Unable to lock user account '%s'. No tools available. " " Tried: %s.") % (name, [c[0] for c in lock_tools])) try: - util.subp(cmd) + subp.subp(cmd) except Exception as e: util.logexc(LOG, 'Failed to disable password for user %s', name) raise e def expire_passwd(self, user): try: - util.subp(['passwd', '--expire', user]) + subp.subp(['passwd', '--expire', user]) except Exception as e: util.logexc(LOG, "Failed to set 'expire' for %s", user) raise e @@ -611,7 +612,7 @@ class Distro(metaclass=abc.ABCMeta): cmd.append('-e') try: - util.subp(cmd, pass_string, logstring="chpasswd for %s" % user) + subp.subp(cmd, pass_string, logstring="chpasswd for %s" % user) except Exception as e: util.logexc(LOG, "Failed to set password for %s", user) raise e @@ -708,7 +709,7 @@ class Distro(metaclass=abc.ABCMeta): LOG.warning("Skipping creation of existing group '%s'", name) else: try: - util.subp(group_add_cmd) + subp.subp(group_add_cmd) LOG.info("Created new group %s", name) except Exception: util.logexc(LOG, "Failed to create group %s", name) @@ -721,7 +722,7 @@ class Distro(metaclass=abc.ABCMeta): "; user does not exist.", member, name) continue - util.subp(['usermod', '-a', '-G', name, member]) + subp.subp(['usermod', '-a', '-G', name, member]) LOG.info("Added user '%s' to group '%s'", member, name) diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 9f89c5f9..038aa9ac 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -8,6 +8,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging from cloudinit import util +from cloudinit import subp from cloudinit.distros import net_util from cloudinit.distros.parsers.hostname import HostnameConf @@ -44,7 +45,7 @@ class Distro(distros.Distro): def apply_locale(self, locale, out_fn=None): if not out_fn: out_fn = self.locale_conf_fn - util.subp(['locale-gen', '-G', locale], capture=False) + subp.subp(['locale-gen', '-G', locale], capture=False) # "" provides trailing newline during join lines = [ util.make_header(), @@ -76,11 +77,11 @@ class Distro(distros.Distro): def _enable_interface(self, device_name): cmd = ['netctl', 'reenable', device_name] try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) def _bring_up_interface(self, device_name): @@ -88,12 +89,12 @@ class Distro(distros.Distro): LOG.debug("Attempting to run bring up interface %s using command %s", device_name, cmd) try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False @@ -158,7 +159,7 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def update_package_sources(self): self._runner.run("update-sources", self.package_command, @@ -173,8 +174,8 @@ def _render_network(entries, target="/", conf_dir="etc/netctl", devs = [] nameservers = [] - resolv_conf = util.target_path(target, resolv_conf) - conf_dir = util.target_path(target, conf_dir) + resolv_conf = subp.target_path(target, resolv_conf) + conf_dir = subp.target_path(target, conf_dir) for (dev, info) in entries.items(): if dev == 'lo': diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py index 37cf93bf..c2d1f77d 100644 --- a/cloudinit/distros/bsd.py +++ b/cloudinit/distros/bsd.py @@ -5,6 +5,7 @@ from cloudinit.distros import bsd_utils from cloudinit import helpers from cloudinit import log as logging from cloudinit import net +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -50,7 +51,7 @@ class BSD(distros.Distro): else: group_add_cmd = self.group_add_cmd_prefix + [name] try: - util.subp(group_add_cmd) + subp.subp(group_add_cmd) LOG.info("Created new group %s", name) except Exception: util.logexc(LOG, "Failed to create group %s", name) @@ -63,7 +64,7 @@ class BSD(distros.Distro): "; user does not exist.", member, name) continue try: - util.subp(self._get_add_member_to_group_cmd(member, name)) + subp.subp(self._get_add_member_to_group_cmd(member, name)) LOG.info("Added user '%s' to group '%s'", member, name) except Exception: util.logexc(LOG, "Failed to add user '%s' to group '%s'", @@ -111,7 +112,7 @@ class BSD(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, env=self._get_pkg_cmd_environ(), capture=False) + subp.subp(cmd, env=self._get_pkg_cmd_environ(), capture=False) def _write_network_config(self, netconfig): return self._supported_write_network_config(netconfig) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 128bb523..844aaf21 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -13,6 +13,7 @@ import os from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros.parsers.hostname import HostnameConf @@ -197,7 +198,7 @@ class Distro(distros.Distro): # Allow the output of this to flow outwards (ie not be captured) util.log_time(logfunc=LOG.debug, msg="apt-%s [%s]" % (command, ' '.join(cmd)), - func=util.subp, + func=subp.subp, args=(cmd,), kwargs={'env': e, 'capture': False}) def update_package_sources(self): @@ -214,7 +215,7 @@ def _get_wrapper_prefix(cmd, mode): if (util.is_true(mode) or (str(mode).lower() == "auto" and cmd[0] and - util.which(cmd[0]))): + subp.which(cmd[0]))): return cmd else: return [] @@ -269,7 +270,7 @@ def update_locale_conf(locale, sys_path, keyname='LANG'): """Update system locale config""" LOG.debug('Updating %s with locale setting %s=%s', sys_path, keyname, locale) - util.subp( + subp.subp( ['update-locale', '--locale-file=' + sys_path, '%s=%s' % (keyname, locale)], capture=False) @@ -291,7 +292,7 @@ def regenerate_locale(locale, sys_path, keyname='LANG'): # finally, trigger regeneration LOG.debug('Generating locales for %s', locale) - util.subp(['locale-gen', locale], capture=False) + subp.subp(['locale-gen', locale], capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index b3a4ad67..dde34d41 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -10,6 +10,7 @@ from io import StringIO import cloudinit.distros.bsd from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -78,7 +79,7 @@ class Distro(cloudinit.distros.bsd.BSD): # Run the command LOG.info("Adding user %s", name) try: - util.subp(pw_useradd_cmd, logstring=log_pw_useradd_cmd) + subp.subp(pw_useradd_cmd, logstring=log_pw_useradd_cmd) except Exception: util.logexc(LOG, "Failed to create user %s", name) raise @@ -90,7 +91,7 @@ class Distro(cloudinit.distros.bsd.BSD): def expire_passwd(self, user): try: - util.subp(['pw', 'usermod', user, '-p', '01-Jan-1970']) + subp.subp(['pw', 'usermod', user, '-p', '01-Jan-1970']) except Exception: util.logexc(LOG, "Failed to set pw expiration for %s", user) raise @@ -102,7 +103,7 @@ class Distro(cloudinit.distros.bsd.BSD): hash_opt = "-h" try: - util.subp(['pw', 'usermod', user, hash_opt, '0'], + subp.subp(['pw', 'usermod', user, hash_opt, '0'], data=passwd, logstring="chpasswd for %s" % user) except Exception: util.logexc(LOG, "Failed to set password for %s", user) @@ -110,7 +111,7 @@ class Distro(cloudinit.distros.bsd.BSD): def lock_passwd(self, name): try: - util.subp(['pw', 'usermod', name, '-h', '-']) + subp.subp(['pw', 'usermod', name, '-h', '-']) except Exception: util.logexc(LOG, "Failed to lock user %s", name) raise @@ -131,8 +132,8 @@ class Distro(cloudinit.distros.bsd.BSD): try: LOG.debug("Running cap_mkdb for %s", locale) - util.subp(['cap_mkdb', self.login_conf_fn]) - except util.ProcessExecutionError: + subp.subp(['cap_mkdb', self.login_conf_fn]) + except subp.ProcessExecutionError: # cap_mkdb failed, so restore the backup. util.logexc(LOG, "Failed to apply locale %s", locale) try: diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index dc57717d..2bee1c89 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -9,6 +9,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros import net_util @@ -39,7 +40,7 @@ class Distro(distros.Distro): def apply_locale(self, locale, out_fn=None): if not out_fn: out_fn = self.locale_conf_fn - util.subp(['locale-gen', '-G', locale], capture=False) + subp.subp(['locale-gen', '-G', locale], capture=False) # "" provides trailing newline during join lines = [ util.make_header(), @@ -94,11 +95,11 @@ class Distro(distros.Distro): cmd = ['rc-update', 'add', 'net.{name}'.format(name=dev), 'default'] try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) @@ -119,12 +120,12 @@ class Distro(distros.Distro): LOG.debug("Attempting to run bring up interface %s using command %s", device_name, cmd) try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False @@ -137,11 +138,11 @@ class Distro(distros.Distro): # Grab device names from init scripts cmd = ['ls', '/etc/init.d/net.*'] try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False devices = [x.split('.')[2] for x in _out.split(' ')] @@ -208,7 +209,7 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def update_package_sources(self): self._runner.run("update-sources", self.package_command, diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py index ecc8239a..066737a8 100644 --- a/cloudinit/distros/netbsd.py +++ b/cloudinit/distros/netbsd.py @@ -8,6 +8,7 @@ import platform import cloudinit.distros.bsd from cloudinit import log as logging +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -76,7 +77,7 @@ class NetBSD(cloudinit.distros.bsd.BSD): # Run the command LOG.info("Adding user %s", name) try: - util.subp(adduser_cmd, logstring=log_adduser_cmd) + subp.subp(adduser_cmd, logstring=log_adduser_cmd) except Exception: util.logexc(LOG, "Failed to create user %s", name) raise @@ -103,7 +104,7 @@ class NetBSD(cloudinit.distros.bsd.BSD): crypt.mksalt(method)) try: - util.subp(['usermod', '-p', hashed_pw, user]) + subp.subp(['usermod', '-p', hashed_pw, user]) except Exception: util.logexc(LOG, "Failed to set password for %s", user) raise @@ -111,21 +112,21 @@ class NetBSD(cloudinit.distros.bsd.BSD): def force_passwd_change(self, user): try: - util.subp(['usermod', '-F', user]) + subp.subp(['usermod', '-F', user]) except Exception: util.logexc(LOG, "Failed to set pw expiration for %s", user) raise def lock_passwd(self, name): try: - util.subp(['usermod', '-C', 'yes', name]) + subp.subp(['usermod', '-C', 'yes', name]) except Exception: util.logexc(LOG, "Failed to lock user %s", name) raise def unlock_passwd(self, name): try: - util.subp(['usermod', '-C', 'no', name]) + subp.subp(['usermod', '-C', 'no', name]) except Exception: util.logexc(LOG, "Failed to unlock user %s", name) raise diff --git a/cloudinit/distros/openbsd.py b/cloudinit/distros/openbsd.py index ca094156..07c76530 100644 --- a/cloudinit/distros/openbsd.py +++ b/cloudinit/distros/openbsd.py @@ -7,6 +7,7 @@ import platform import cloudinit.distros.netbsd from cloudinit import log as logging +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -27,7 +28,7 @@ class Distro(cloudinit.distros.netbsd.NetBSD): def lock_passwd(self, name): try: - util.subp(['usermod', '-p', '*', name]) + subp.subp(['usermod', '-p', '*', name]) except Exception: util.logexc(LOG, "Failed to lock user %s", name) raise diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py index 68028d20..ffb7d0e8 100644 --- a/cloudinit/distros/opensuse.py +++ b/cloudinit/distros/opensuse.py @@ -14,6 +14,7 @@ from cloudinit.distros.parsers.hostname import HostnameConf from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros import rhel_util as rhutil @@ -97,7 +98,7 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def set_timezone(self, tz): tz_file = self._find_tz_file(tz) @@ -129,7 +130,7 @@ class Distro(distros.Distro): if self.uses_systemd() and filename.endswith('/previous-hostname'): return util.load_file(filename).strip() elif self.uses_systemd(): - (out, _err) = util.subp(['hostname']) + (out, _err) = subp.subp(['hostname']) if len(out): return out else: @@ -163,7 +164,7 @@ class Distro(distros.Distro): if self.uses_systemd() and out_fn.endswith('/previous-hostname'): util.write_file(out_fn, hostname) elif self.uses_systemd(): - util.subp(['hostnamectl', 'set-hostname', str(hostname)]) + subp.subp(['hostnamectl', 'set-hostname', str(hostname)]) else: conf = None try: diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index f55d96f7..c72f7c17 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -11,6 +11,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros import rhel_util @@ -83,7 +84,7 @@ class Distro(distros.Distro): if self.uses_systemd() and out_fn.endswith('/previous-hostname'): util.write_file(out_fn, hostname) elif self.uses_systemd(): - util.subp(['hostnamectl', 'set-hostname', str(hostname)]) + subp.subp(['hostnamectl', 'set-hostname', str(hostname)]) else: host_cfg = { 'HOSTNAME': hostname, @@ -108,7 +109,7 @@ class Distro(distros.Distro): if self.uses_systemd() and filename.endswith('/previous-hostname'): return util.load_file(filename).strip() elif self.uses_systemd(): - (out, _err) = util.subp(['hostname']) + (out, _err) = subp.subp(['hostname']) if len(out): return out else: @@ -146,7 +147,7 @@ class Distro(distros.Distro): if pkgs is None: pkgs = [] - if util.which('dnf'): + if subp.which('dnf'): LOG.debug('Using DNF for package management') cmd = ['dnf'] else: @@ -173,7 +174,7 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def update_package_sources(self): self._runner.run("update-sources", self.package_command, diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py index 7fe17a2e..72b5ac59 100644 --- a/cloudinit/gpg.py +++ b/cloudinit/gpg.py @@ -8,7 +8,7 @@ """gpg.py - Collection of gpg key related functions""" from cloudinit import log as logging -from cloudinit import util +from cloudinit import subp import time @@ -18,9 +18,9 @@ LOG = logging.getLogger(__name__) def export_armour(key): """Export gpg key, armoured key gets returned""" try: - (armour, _) = util.subp(["gpg", "--export", "--armour", key], + (armour, _) = subp.subp(["gpg", "--export", "--armour", key], capture=True) - except util.ProcessExecutionError as error: + except subp.ProcessExecutionError as error: # debug, since it happens for any key not on the system initially LOG.debug('Failed to export armoured key "%s": %s', key, error) armour = None @@ -51,11 +51,11 @@ def recv_key(key, keyserver, retries=(1, 1)): while True: trynum += 1 try: - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) LOG.debug("Imported key '%s' from keyserver '%s' on try %d", key, keyserver, trynum) return - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: error = e try: naplen = next(sleeps) @@ -72,9 +72,9 @@ def recv_key(key, keyserver, retries=(1, 1)): def delete_key(key): """Delete the specified key from the local gpg ring""" try: - util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], + subp.subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=True) - except util.ProcessExecutionError as error: + except subp.ProcessExecutionError as error: LOG.warning('Failed delete key "%s": %s', key, error) diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index dca50a49..c6205097 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -12,6 +12,7 @@ import os from cloudinit import handlers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.settings import (PER_ALWAYS) @@ -48,8 +49,8 @@ class BootHookPartHandler(handlers.Handler): env = os.environ.copy() if self.instance_id is not None: env['INSTANCE_ID'] = str(self.instance_id) - util.subp([filepath], env=env) - except util.ProcessExecutionError: + subp.subp([filepath], env=env) + except subp.ProcessExecutionError: util.logexc(LOG, "Boothooks script %s execution error", filepath) except Exception: util.logexc(LOG, "Boothooks unknown error when running %s", diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index 003cad60..a9d29537 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -13,6 +13,7 @@ import re from cloudinit import handlers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.settings import (PER_INSTANCE) @@ -52,7 +53,7 @@ class UpstartJobPartHandler(handlers.Handler): util.write_file(path, payload, 0o644) if SUITABLE_UPSTART: - util.subp(["initctl", "reload-configuration"], capture=False) + subp.subp(["initctl", "reload-configuration"], capture=False) def _has_suitable_upstart(): @@ -63,7 +64,7 @@ def _has_suitable_upstart(): if not os.path.exists("/sbin/initctl"): return False try: - (version_out, _err) = util.subp(["initctl", "version"]) + (version_out, _err) = subp.subp(["initctl", "version"]) except Exception: util.logexc(LOG, "initctl version failed") return False @@ -77,7 +78,7 @@ def _has_suitable_upstart(): if not os.path.exists("/usr/bin/dpkg-query"): return False try: - (dpkg_ver, _err) = util.subp(["dpkg-query", + (dpkg_ver, _err) = subp.subp(["dpkg-query", "--showformat=${Version}", "--show", "upstart"], rcs=[0, 1]) except Exception: @@ -86,9 +87,9 @@ def _has_suitable_upstart(): try: good = "1.8-0ubuntu1.2" - util.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good]) + subp.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good]) return True - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code == 1: pass else: diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 8af24fa9..a57fea0a 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -12,6 +12,7 @@ import os import re from functools import partial +from cloudinit import subp from cloudinit import util from cloudinit.net.network_state import mask_to_net_prefix from cloudinit.url_helper import UrlError, readurl @@ -358,7 +359,7 @@ def find_fallback_nic_on_freebsd(blacklist_drivers=None): we'll use the first interface from ``ifconfig -l -u ether`` """ - stdout, _stderr = util.subp(['ifconfig', '-l', '-u', 'ether']) + stdout, _stderr = subp.subp(['ifconfig', '-l', '-u', 'ether']) values = stdout.split() if values: return values[0] @@ -620,9 +621,9 @@ def _get_current_rename_info(check_downable=True): if check_downable: nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]") - ipv6, _err = util.subp(['ip', '-6', 'addr', 'show', 'permanent', + ipv6, _err = subp.subp(['ip', '-6', 'addr', 'show', 'permanent', 'scope', 'global'], capture=True) - ipv4, _err = util.subp(['ip', '-4', 'addr', 'show'], capture=True) + ipv4, _err = subp.subp(['ip', '-4', 'addr', 'show'], capture=True) nics_with_addresses = set() for bytes_out in (ipv6, ipv4): @@ -658,13 +659,13 @@ def _rename_interfaces(renames, strict_present=True, strict_busy=True, for data in cur_info.values()) def rename(cur, new): - util.subp(["ip", "link", "set", cur, "name", new], capture=True) + subp.subp(["ip", "link", "set", cur, "name", new], capture=True) def down(name): - util.subp(["ip", "link", "set", name, "down"], capture=True) + subp.subp(["ip", "link", "set", name, "down"], capture=True) def up(name): - util.subp(["ip", "link", "set", name, "up"], capture=True) + subp.subp(["ip", "link", "set", name, "up"], capture=True) ops = [] errors = [] @@ -819,7 +820,7 @@ def get_interfaces_by_mac(): def get_interfaces_by_mac_on_freebsd(): - (out, _) = util.subp(['ifconfig', '-a', 'ether']) + (out, _) = subp.subp(['ifconfig', '-a', 'ether']) # flatten each interface block in a single line def flatten(out): @@ -850,7 +851,7 @@ def get_interfaces_by_mac_on_netbsd(): re_field_match = ( r"(?P\w+).*address:\s" r"(?P([\da-f]{2}[:-]){5}([\da-f]{2})).*") - (out, _) = util.subp(['ifconfig', '-a']) + (out, _) = subp.subp(['ifconfig', '-a']) if_lines = re.sub(r'\n\s+', ' ', out).splitlines() for line in if_lines: m = re.match(re_field_match, line) @@ -865,7 +866,7 @@ def get_interfaces_by_mac_on_openbsd(): re_field_match = ( r"(?P\w+).*lladdr\s" r"(?P([\da-f]{2}[:-]){5}([\da-f]{2})).*") - (out, _) = util.subp(['ifconfig', '-a']) + (out, _) = subp.subp(['ifconfig', '-a']) if_lines = re.sub(r'\n\s+', ' ', out).splitlines() for line in if_lines: m = re.match(re_field_match, line) @@ -1067,11 +1068,11 @@ class EphemeralIPv4Network(object): def __exit__(self, excp_type, excp_value, excp_traceback): """Teardown anything we set up.""" for cmd in self.cleanup_cmds: - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def _delete_address(self, address, prefix): """Perform the ip command to remove the specified address.""" - util.subp( + subp.subp( ['ip', '-family', 'inet', 'addr', 'del', '%s/%s' % (address, prefix), 'dev', self.interface], capture=True) @@ -1083,11 +1084,11 @@ class EphemeralIPv4Network(object): 'Attempting setup of ephemeral network on %s with %s brd %s', self.interface, cidr, self.broadcast) try: - util.subp( + subp.subp( ['ip', '-family', 'inet', 'addr', 'add', cidr, 'broadcast', self.broadcast, 'dev', self.interface], capture=True, update_env={'LANG': 'C'}) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if "File exists" not in e.stderr: raise LOG.debug( @@ -1095,7 +1096,7 @@ class EphemeralIPv4Network(object): self.interface, self.ip) else: # Address creation success, bring up device and queue cleanup - util.subp( + subp.subp( ['ip', '-family', 'inet', 'link', 'set', 'dev', self.interface, 'up'], capture=True) self.cleanup_cmds.append( @@ -1112,7 +1113,7 @@ class EphemeralIPv4Network(object): via_arg = [] if gateway != "0.0.0.0/0": via_arg = ['via', gateway] - util.subp( + subp.subp( ['ip', '-4', 'route', 'add', net_address] + via_arg + ['dev', self.interface], capture=True) self.cleanup_cmds.insert( @@ -1122,20 +1123,20 @@ class EphemeralIPv4Network(object): def _bringup_router(self): """Perform the ip commands to fully setup the router if needed.""" # Check if a default route exists and exit if it does - out, _ = util.subp(['ip', 'route', 'show', '0.0.0.0/0'], capture=True) + out, _ = subp.subp(['ip', 'route', 'show', '0.0.0.0/0'], capture=True) if 'default' in out: LOG.debug( 'Skip ephemeral route setup. %s already has default route: %s', self.interface, out.strip()) return - util.subp( + subp.subp( ['ip', '-4', 'route', 'add', self.router, 'dev', self.interface, 'src', self.ip], capture=True) self.cleanup_cmds.insert( 0, ['ip', '-4', 'route', 'del', self.router, 'dev', self.interface, 'src', self.ip]) - util.subp( + subp.subp( ['ip', '-4', 'route', 'add', 'default', 'via', self.router, 'dev', self.interface], capture=True) self.cleanup_cmds.insert( diff --git a/cloudinit/net/bsd.py b/cloudinit/net/bsd.py index fb714d4c..1c355a98 100644 --- a/cloudinit/net/bsd.py +++ b/cloudinit/net/bsd.py @@ -5,6 +5,7 @@ import re from cloudinit import log as logging from cloudinit import net from cloudinit import util +from cloudinit import subp from cloudinit.distros.parsers.resolv_conf import ResolvConf from cloudinit.distros import bsd_utils @@ -18,11 +19,11 @@ class BSDRenderer(renderer.Renderer): rc_conf_fn = 'etc/rc.conf' def get_rc_config_value(self, key): - fn = util.target_path(self.target, self.rc_conf_fn) + fn = subp.target_path(self.target, self.rc_conf_fn) bsd_utils.get_rc_config_value(key, fn=fn) def set_rc_config_value(self, key, value): - fn = util.target_path(self.target, self.rc_conf_fn) + fn = subp.target_path(self.target, self.rc_conf_fn) bsd_utils.set_rc_config_value(key, value, fn=fn) def __init__(self, config=None): @@ -111,12 +112,12 @@ class BSDRenderer(renderer.Renderer): # Try to read the /etc/resolv.conf or just start from scratch if that # fails. try: - resolvconf = ResolvConf(util.load_file(util.target_path( + resolvconf = ResolvConf(util.load_file(subp.target_path( target, self.resolv_conf_fn))) resolvconf.parse() except IOError: util.logexc(LOG, "Failed to parse %s, use new empty file", - util.target_path(target, self.resolv_conf_fn)) + subp.target_path(target, self.resolv_conf_fn)) resolvconf = ResolvConf('') resolvconf.parse() @@ -134,7 +135,7 @@ class BSDRenderer(renderer.Renderer): except ValueError: util.logexc(LOG, "Failed to add search domain %s", domain) util.write_file( - util.target_path(target, self.resolv_conf_fn), + subp.target_path(target, self.resolv_conf_fn), str(resolvconf), 0o644) def render_network_state(self, network_state, templates=None, target=None): diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 19d0199c..d03baeab 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -17,6 +17,7 @@ from cloudinit.net import ( has_url_connectivity) from cloudinit.net.network_state import mask_and_ipv4_to_bcast_addr as bcip from cloudinit import temp_utils +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -150,7 +151,7 @@ def maybe_perform_dhcp_discovery(nic=None): LOG.debug( 'Skip dhcp_discovery: nic %s not found in get_devicelist.', nic) return [] - dhclient_path = util.which('dhclient') + dhclient_path = subp.which('dhclient') if not dhclient_path: LOG.debug('Skip dhclient configuration: No dhclient command found.') return [] @@ -219,10 +220,10 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir): # Generally dhclient relies on dhclient-script PREINIT action to bring the # link up before attempting discovery. Since we are using -sf /bin/true, # we need to do that "link up" ourselves first. - util.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True) + subp.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True) cmd = [sandbox_dhclient_cmd, '-1', '-v', '-lf', lease_file, '-pf', pid_file, interface, '-sf', '/bin/true'] - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) # Wait for pid file and lease file to appear, and for the process # named by the pid file to daemonize (have pid 1 as its parent). If we diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 2f714563..b4c69457 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -11,6 +11,7 @@ from . import renderer from .network_state import subnet_is_ipv6 from cloudinit import log as logging +from cloudinit import subp from cloudinit import util @@ -511,13 +512,13 @@ class Renderer(renderer.Renderer): return '\n\n'.join(['\n'.join(s) for s in sections]) + "\n" def render_network_state(self, network_state, templates=None, target=None): - fpeni = util.target_path(target, self.eni_path) + fpeni = subp.target_path(target, self.eni_path) util.ensure_dir(os.path.dirname(fpeni)) header = self.eni_header if self.eni_header else "" util.write_file(fpeni, header + self._render_interfaces(network_state)) if self.netrules_path: - netrules = util.target_path(target, self.netrules_path) + netrules = subp.target_path(target, self.netrules_path) util.ensure_dir(os.path.dirname(netrules)) util.write_file(netrules, self._render_persistent_net(network_state)) @@ -544,9 +545,9 @@ def available(target=None): expected = ['ifquery', 'ifup', 'ifdown'] search = ['/sbin', '/usr/sbin'] for p in expected: - if not util.which(p, search=search, target=target): + if not subp.which(p, search=search, target=target): return False - eni = util.target_path(target, 'etc/network/interfaces') + eni = subp.target_path(target, 'etc/network/interfaces') if not os.path.isfile(eni): return False diff --git a/cloudinit/net/freebsd.py b/cloudinit/net/freebsd.py index 60f05bb2..0285dfec 100644 --- a/cloudinit/net/freebsd.py +++ b/cloudinit/net/freebsd.py @@ -2,6 +2,7 @@ from cloudinit import log as logging import cloudinit.net.bsd +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -30,17 +31,17 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): LOG.debug("freebsd generate postcmd disabled") return - util.subp(['service', 'netif', 'restart'], capture=True) + subp.subp(['service', 'netif', 'restart'], capture=True) # On FreeBSD 10, the restart of routing and dhclient is likely to fail # because # - routing: it cannot remove the loopback route, but it will still set # up the default route as expected. # - dhclient: it cannot stop the dhclient started by the netif service. # In both case, the situation is ok, and we can proceed. - util.subp(['service', 'routing', 'restart'], capture=True, rcs=[0, 1]) + subp.subp(['service', 'routing', 'restart'], capture=True, rcs=[0, 1]) for dhcp_interface in self.dhcp_interfaces(): - util.subp(['service', 'dhclient', 'restart', dhcp_interface], + subp.subp(['service', 'dhclient', 'restart', dhcp_interface], rcs=[0, 1], capture=True) diff --git a/cloudinit/net/netbsd.py b/cloudinit/net/netbsd.py index 9cc8ef31..30437b5f 100644 --- a/cloudinit/net/netbsd.py +++ b/cloudinit/net/netbsd.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import log as logging +from cloudinit import subp from cloudinit import util import cloudinit.net.bsd @@ -29,9 +30,9 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): LOG.debug("netbsd generate postcmd disabled") return - util.subp(['service', 'network', 'restart'], capture=True) + subp.subp(['service', 'network', 'restart'], capture=True) if self.dhcp_interfaces(): - util.subp(['service', 'dhcpcd', 'restart'], capture=True) + subp.subp(['service', 'dhcpcd', 'restart'], capture=True) def set_route(self, network, netmask, gateway): if network == '0.0.0.0': diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 89855270..53347c83 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -8,6 +8,7 @@ from .network_state import subnet_is_ipv6, NET_CONFIG_TO_V2, IPV6_DYNAMIC_TYPES from cloudinit import log as logging from cloudinit import util +from cloudinit import subp from cloudinit import safeyaml from cloudinit.net import SYS_CLASS_NET, get_devicelist @@ -164,14 +165,14 @@ def _extract_bond_slaves_by_name(interfaces, entry, bond_master): def _clean_default(target=None): # clean out any known default files and derived files in target # LP: #1675576 - tpath = util.target_path(target, "etc/netplan/00-snapd-config.yaml") + tpath = subp.target_path(target, "etc/netplan/00-snapd-config.yaml") if not os.path.isfile(tpath): return content = util.load_file(tpath, decode=False) if content != KNOWN_SNAPD_CONFIG: return - derived = [util.target_path(target, f) for f in ( + derived = [subp.target_path(target, f) for f in ( 'run/systemd/network/10-netplan-all-en.network', 'run/systemd/network/10-netplan-all-eth.network', 'run/systemd/generator/netplan.stamp')] @@ -203,10 +204,10 @@ class Renderer(renderer.Renderer): def features(self): if self._features is None: try: - info_blob, _err = util.subp(self.NETPLAN_INFO, capture=True) + info_blob, _err = subp.subp(self.NETPLAN_INFO, capture=True) info = util.load_yaml(info_blob) self._features = info['netplan.io']['features'] - except util.ProcessExecutionError: + except subp.ProcessExecutionError: # if the info subcommand is not present then we don't have any # new features pass @@ -218,7 +219,7 @@ class Renderer(renderer.Renderer): # check network state for version # if v2, then extract network_state.config # else render_v2_from_state - fpnplan = os.path.join(util.target_path(target), self.netplan_path) + fpnplan = os.path.join(subp.target_path(target), self.netplan_path) util.ensure_dir(os.path.dirname(fpnplan)) header = self.netplan_header if self.netplan_header else "" @@ -239,7 +240,7 @@ class Renderer(renderer.Renderer): if not run: LOG.debug("netplan generate postcmd disabled") return - util.subp(self.NETPLAN_GENERATE, capture=True) + subp.subp(self.NETPLAN_GENERATE, capture=True) def _net_setup_link(self, run=False): """To ensure device link properties are applied, we poke @@ -253,7 +254,7 @@ class Renderer(renderer.Renderer): for cmd in [setup_lnk + [SYS_CLASS_NET + iface] for iface in get_devicelist() if os.path.islink(SYS_CLASS_NET + iface)]: - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def _render_content(self, network_state): @@ -406,7 +407,7 @@ def available(target=None): expected = ['netplan'] search = ['/usr/sbin', '/sbin'] for p in expected: - if not util.which(p, search=search, target=target): + if not subp.which(p, search=search, target=target): return False return True diff --git a/cloudinit/net/openbsd.py b/cloudinit/net/openbsd.py index b9897e90..489ea48b 100644 --- a/cloudinit/net/openbsd.py +++ b/cloudinit/net/openbsd.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import log as logging +from cloudinit import subp from cloudinit import util import cloudinit.net.bsd @@ -12,7 +13,7 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): def write_config(self): for device_name, v in self.interface_configurations.items(): if_file = 'etc/hostname.{}'.format(device_name) - fn = util.target_path(self.target, if_file) + fn = subp.target_path(self.target, if_file) if device_name in self.dhcp_interfaces(): content = 'dhcp\n' elif isinstance(v, dict): @@ -30,12 +31,12 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): if not self._postcmds: LOG.debug("openbsd generate postcmd disabled") return - util.subp(['sh', '/etc/netstart'], capture=True) + subp.subp(['sh', '/etc/netstart'], capture=True) def set_route(self, network, netmask, gateway): if network == '0.0.0.0': if_file = 'etc/mygate' - fn = util.target_path(self.target, if_file) + fn = subp.target_path(self.target, if_file) content = gateway + '\n' util.write_file(fn, content) diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 0a387377..f36c300f 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -9,6 +9,7 @@ from configobj import ConfigObj from cloudinit import log as logging from cloudinit import util +from cloudinit import subp from cloudinit.distros.parsers import networkmanager_conf from cloudinit.distros.parsers import resolv_conf @@ -858,19 +859,19 @@ class Renderer(renderer.Renderer): if not templates: templates = self.templates file_mode = 0o644 - base_sysconf_dir = util.target_path(target, self.sysconf_dir) + base_sysconf_dir = subp.target_path(target, self.sysconf_dir) for path, data in self._render_sysconfig(base_sysconf_dir, network_state, self.flavor, templates=templates).items(): util.write_file(path, data, file_mode) if self.dns_path: - dns_path = util.target_path(target, self.dns_path) + dns_path = subp.target_path(target, self.dns_path) resolv_content = self._render_dns(network_state, existing_dns_path=dns_path) if resolv_content: util.write_file(dns_path, resolv_content, file_mode) if self.networkmanager_conf_path: - nm_conf_path = util.target_path(target, + nm_conf_path = subp.target_path(target, self.networkmanager_conf_path) nm_conf_content = self._render_networkmanager_conf(network_state, templates) @@ -878,12 +879,12 @@ class Renderer(renderer.Renderer): util.write_file(nm_conf_path, nm_conf_content, file_mode) if self.netrules_path: netrules_content = self._render_persistent_net(network_state) - netrules_path = util.target_path(target, self.netrules_path) + netrules_path = subp.target_path(target, self.netrules_path) util.write_file(netrules_path, netrules_content, file_mode) if available_nm(target=target): - enable_ifcfg_rh(util.target_path(target, path=NM_CFG_FILE)) + enable_ifcfg_rh(subp.target_path(target, path=NM_CFG_FILE)) - sysconfig_path = util.target_path(target, templates.get('control')) + sysconfig_path = subp.target_path(target, templates.get('control')) # Distros configuring /etc/sysconfig/network as a file e.g. Centos if sysconfig_path.endswith('network'): util.ensure_dir(os.path.dirname(sysconfig_path)) @@ -906,20 +907,20 @@ def available_sysconfig(target=None): expected = ['ifup', 'ifdown'] search = ['/sbin', '/usr/sbin'] for p in expected: - if not util.which(p, search=search, target=target): + if not subp.which(p, search=search, target=target): return False expected_paths = [ 'etc/sysconfig/network-scripts/network-functions', 'etc/sysconfig/config'] for p in expected_paths: - if os.path.isfile(util.target_path(target, p)): + if os.path.isfile(subp.target_path(target, p)): return True return False def available_nm(target=None): - if not os.path.isfile(util.target_path(target, path=NM_CFG_FILE)): + if not os.path.isfile(subp.target_path(target, path=NM_CFG_FILE)): return False return True diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index 7768da7c..d4881592 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -266,7 +266,7 @@ class TestDHCPDiscoveryClean(CiTestCase): 'Skip dhcp_discovery: nic idontexist not found in get_devicelist.', self.logs.getvalue()) - @mock.patch('cloudinit.net.dhcp.util.which') + @mock.patch('cloudinit.net.dhcp.subp.which') @mock.patch('cloudinit.net.dhcp.find_fallback_nic') def test_absent_dhclient_command(self, m_fallback, m_which): """When dhclient doesn't exist in the OS, log the issue and no-op.""" @@ -279,7 +279,7 @@ class TestDHCPDiscoveryClean(CiTestCase): @mock.patch('cloudinit.temp_utils.os.getuid') @mock.patch('cloudinit.net.dhcp.dhcp_discovery') - @mock.patch('cloudinit.net.dhcp.util.which') + @mock.patch('cloudinit.net.dhcp.subp.which') @mock.patch('cloudinit.net.dhcp.find_fallback_nic') def test_dhclient_run_with_tmpdir(self, m_fback, m_which, m_dhcp, m_uid): """maybe_perform_dhcp_discovery passes tmpdir to dhcp_discovery.""" @@ -302,7 +302,7 @@ class TestDHCPDiscoveryClean(CiTestCase): @mock.patch('time.sleep', mock.MagicMock()) @mock.patch('cloudinit.net.dhcp.os.kill') - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') def test_dhcp_discovery_run_in_sandbox_warns_invalid_pid(self, m_subp, m_kill): """dhcp_discovery logs a warning when pidfile contains invalid content. @@ -337,7 +337,7 @@ class TestDHCPDiscoveryClean(CiTestCase): @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid') @mock.patch('cloudinit.net.dhcp.os.kill') @mock.patch('cloudinit.net.dhcp.util.wait_for_files') - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') def test_dhcp_discovery_run_in_sandbox_waits_on_lease_and_pid(self, m_subp, m_wait, @@ -364,7 +364,7 @@ class TestDHCPDiscoveryClean(CiTestCase): @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid') @mock.patch('cloudinit.net.dhcp.os.kill') - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') def test_dhcp_discovery_run_in_sandbox(self, m_subp, m_kill, m_getppid): """dhcp_discovery brings up the interface and runs dhclient. @@ -529,7 +529,7 @@ class TestEphemeralDhcpNoNetworkSetup(HttprettyTestCase): # Ensure that no teardown happens: m_dhcp.assert_not_called() - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') def test_ephemeral_dhcp_setup_network_if_url_connectivity( self, m_dhcp, m_subp): diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index 835ed807..1c0a16a8 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -14,7 +14,8 @@ import requests import cloudinit.net as net from cloudinit import safeyaml as yaml from cloudinit.tests.helpers import CiTestCase, HttprettyTestCase -from cloudinit.util import ProcessExecutionError, ensure_file, write_file +from cloudinit.subp import ProcessExecutionError +from cloudinit.util import ensure_file, write_file class TestSysDevPath(CiTestCase): @@ -541,7 +542,7 @@ class TestInterfaceHasOwnMAC(CiTestCase): net.interface_has_own_mac('eth1', strict=True) -@mock.patch('cloudinit.net.util.subp') +@mock.patch('cloudinit.net.subp.subp') class TestEphemeralIPV4Network(CiTestCase): with_logs = True diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 1001f149..628e2908 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -13,6 +13,7 @@ import re from cloudinit import log as logging from cloudinit.net.network_state import net_prefix_to_ipv4_mask +from cloudinit import subp from cloudinit import util from cloudinit.simpletable import SimpleTable @@ -197,15 +198,15 @@ def _netdev_info_ifconfig(ifconfig_data): def netdev_info(empty=""): devs = {} if util.is_NetBSD(): - (ifcfg_out, _err) = util.subp(["ifconfig", "-a"], rcs=[0, 1]) + (ifcfg_out, _err) = subp.subp(["ifconfig", "-a"], rcs=[0, 1]) devs = _netdev_info_ifconfig_netbsd(ifcfg_out) - elif util.which('ip'): + elif subp.which('ip'): # Try iproute first of all - (ipaddr_out, _err) = util.subp(["ip", "addr", "show"]) + (ipaddr_out, _err) = subp.subp(["ip", "addr", "show"]) devs = _netdev_info_iproute(ipaddr_out) - elif util.which('ifconfig'): + elif subp.which('ifconfig'): # Fall back to net-tools if iproute2 is not present - (ifcfg_out, _err) = util.subp(["ifconfig", "-a"], rcs=[0, 1]) + (ifcfg_out, _err) = subp.subp(["ifconfig", "-a"], rcs=[0, 1]) devs = _netdev_info_ifconfig(ifcfg_out) else: LOG.warning( @@ -285,10 +286,10 @@ def _netdev_route_info_iproute(iproute_data): entry['flags'] = ''.join(flags) routes['ipv4'].append(entry) try: - (iproute_data6, _err6) = util.subp( + (iproute_data6, _err6) = subp.subp( ["ip", "--oneline", "-6", "route", "list", "table", "all"], rcs=[0, 1]) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass else: entries6 = iproute_data6.splitlines() @@ -357,9 +358,9 @@ def _netdev_route_info_netstat(route_data): routes['ipv4'].append(entry) try: - (route_data6, _err6) = util.subp( + (route_data6, _err6) = subp.subp( ["netstat", "-A", "inet6", "--route", "--numeric"], rcs=[0, 1]) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass else: entries6 = route_data6.splitlines() @@ -393,13 +394,13 @@ def _netdev_route_info_netstat(route_data): def route_info(): routes = {} - if util.which('ip'): + if subp.which('ip'): # Try iproute first of all - (iproute_out, _err) = util.subp(["ip", "-o", "route", "list"]) + (iproute_out, _err) = subp.subp(["ip", "-o", "route", "list"]) routes = _netdev_route_info_iproute(iproute_out) - elif util.which('netstat'): + elif subp.which('netstat'): # Fall back to net-tools if iproute2 is not present - (route_out, _err) = util.subp( + (route_out, _err) = subp.subp( ["netstat", "--route", "--numeric", "--extend"], rcs=[0, 1]) routes = _netdev_route_info_netstat(route_out) else: diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index 5270fda8..ac3ecc3d 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -18,9 +18,9 @@ import os.path from cloudinit import log as logging from cloudinit import sources +from cloudinit import subp from cloudinit import util -from cloudinit.util import ProcessExecutionError LOG = logging.getLogger(__name__) @@ -192,7 +192,7 @@ class DataSourceAltCloud(sources.DataSource): # modprobe floppy try: modprobe_floppy() - except ProcessExecutionError as e: + except subp.ProcessExecutionError as e: util.logexc(LOG, 'Failed modprobe: %s', e) return False @@ -201,7 +201,7 @@ class DataSourceAltCloud(sources.DataSource): # udevadm settle for floppy device try: util.udevadm_settle(exists=floppy_dev, timeout=5) - except (ProcessExecutionError, OSError) as e: + except (subp.ProcessExecutionError, OSError) as e: util.logexc(LOG, 'Failed udevadm_settle: %s\n', e) return False @@ -261,7 +261,7 @@ class DataSourceAltCloud(sources.DataSource): def modprobe_floppy(): - out, _err = util.subp(CMD_PROBE_FLOPPY) + out, _err = subp.subp(CMD_PROBE_FLOPPY) LOG.debug('Command: %s\nOutput%s', ' '.join(CMD_PROBE_FLOPPY), out) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 01d9adf2..89312b9e 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -22,6 +22,7 @@ from cloudinit.event import EventType from cloudinit.net.dhcp import EphemeralDHCPv4 from cloudinit import sources from cloudinit.sources.helpers import netlink +from cloudinit import subp from cloudinit.url_helper import UrlError, readurl, retry_on_url_exc from cloudinit import util from cloudinit.reporting import events @@ -139,8 +140,8 @@ def find_dev_from_busdev(camcontrol_out, busdev): def execute_or_debug(cmd, fail_ret=None): try: - return util.subp(cmd)[0] - except util.ProcessExecutionError: + return subp.subp(cmd)[0] + except subp.ProcessExecutionError: LOG.debug("Failed to execute: %s", ' '.join(cmd)) return fail_ret @@ -252,11 +253,11 @@ DEF_PASSWD_REDACTION = 'REDACTED' def get_hostname(hostname_command='hostname'): if not isinstance(hostname_command, (list, tuple)): hostname_command = (hostname_command,) - return util.subp(hostname_command, capture=True)[0].strip() + return subp.subp(hostname_command, capture=True)[0].strip() def set_hostname(hostname, hostname_command='hostname'): - util.subp([hostname_command, hostname]) + subp.subp([hostname_command, hostname]) @azure_ds_telemetry_reporter @@ -343,7 +344,7 @@ class DataSourceAzure(sources.DataSource): try: invoke_agent(agent_cmd) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: # claim the datasource even if the command failed util.logexc(LOG, "agent command '%s' failed.", self.ds_cfg['agent_command']) @@ -982,7 +983,7 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname): if command == "builtin": if util.is_FreeBSD(): command = BOUNCE_COMMAND_FREEBSD - elif util.which('ifup'): + elif subp.which('ifup'): command = BOUNCE_COMMAND_IFUP else: LOG.debug( @@ -993,7 +994,7 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname): shell = not isinstance(command, (list, tuple)) # capture=False, see comments in bug 1202758 and bug 1206164. util.log_time(logfunc=LOG.debug, msg="publishing hostname", - get_uptime=True, func=util.subp, + get_uptime=True, func=subp.subp, kwargs={'args': command, 'shell': shell, 'capture': False, 'env': env}) return True @@ -1003,7 +1004,7 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname): def crtfile_to_pubkey(fname, data=None): pipeline = ('openssl x509 -noout -pubkey < "$0" |' 'ssh-keygen -i -m PKCS8 -f /dev/stdin') - (out, _err) = util.subp(['sh', '-c', pipeline, fname], + (out, _err) = subp.subp(['sh', '-c', pipeline, fname], capture=True, data=data) return out.rstrip() @@ -1015,7 +1016,7 @@ def pubkeys_from_crt_files(flist): for fname in flist: try: pubkeys.append(crtfile_to_pubkey(fname)) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: errors.append(fname) if errors: @@ -1057,7 +1058,7 @@ def invoke_agent(cmd): # this is a function itself to simplify patching it for test if cmd: LOG.debug("invoking agent: %s", cmd) - util.subp(cmd, shell=(not isinstance(cmd, list))) + subp.subp(cmd, shell=(not isinstance(cmd, list))) else: LOG.debug("not invoking agent") diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 2013bed7..54810439 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -22,6 +22,7 @@ from cloudinit import log as logging from cloudinit.net import dhcp from cloudinit import sources from cloudinit import url_helper as uhelp +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -46,7 +47,7 @@ class CloudStackPasswordServerClient(object): # The password server was in the past, a broken HTTP server, but is now # fixed. wget handles this seamlessly, so it's easier to shell out to # that rather than write our own handling code. - output, _ = util.subp([ + output, _ = subp.subp([ 'wget', '--quiet', '--tries', '3', '--timeout', '20', '--output-document', '-', '--header', 'DomU_Request: {0}'.format(domu_request), diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index ae31934b..62756cf7 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -10,6 +10,7 @@ import os from cloudinit import log as logging from cloudinit import sources +from cloudinit import subp from cloudinit import util from cloudinit.net import eni @@ -245,7 +246,7 @@ def find_candidate_devs(probe_optical=True, dslist=None): for device in OPTICAL_DEVICES: try: util.find_devs_with(path=device) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass by_fstype = [] diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py index e0c714e8..d2aa9a58 100644 --- a/cloudinit/sources/DataSourceIBMCloud.py +++ b/cloudinit/sources/DataSourceIBMCloud.py @@ -99,6 +99,7 @@ import os from cloudinit import log as logging from cloudinit import sources from cloudinit.sources.helpers import openstack +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -240,7 +241,7 @@ def get_ibm_platform(): fslabels = {} try: devs = util.blkid() - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: LOG.warning("Failed to run blkid: %s", e) return (None, None) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 41f999e3..4c5eee4f 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -16,6 +16,7 @@ from xml.dom import minidom from cloudinit import log as logging from cloudinit import sources +from cloudinit import subp from cloudinit import util from cloudinit.sources.helpers.vmware.imc.config \ import Config @@ -536,15 +537,15 @@ def transport_iso9660(require_iso=True): def transport_vmware_guestinfo(): rpctool = "vmware-rpctool" not_found = None - if not util.which(rpctool): + if not subp.which(rpctool): return not_found cmd = [rpctool, "info-get guestinfo.ovfEnv"] try: - out, _err = util.subp(cmd) + out, _err = subp.subp(cmd) if out: return out LOG.debug("cmd %s exited 0 with empty stdout: %s", cmd, out) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code != 1: LOG.warning("%s exited with code %d", rpctool, e.exit_code) LOG.debug(e) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index a08ab404..c7fdc2a5 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -21,6 +21,7 @@ import string from cloudinit import log as logging from cloudinit import net from cloudinit import sources +from cloudinit import subp from cloudinit import util @@ -334,7 +335,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None, cmd.extend(bash) - (output, _error) = util.subp(cmd, data=bcmd) + (output, _error) = subp.subp(cmd, data=bcmd) # exclude vars in bash that change on their own or that we used excluded = ( @@ -396,7 +397,7 @@ def read_context_disk_dir(source_dir, asuser=None): path = os.path.join(source_dir, 'context.sh') content = util.load_file(path) context = parse_shell_config(content, asuser=asuser) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: raise BrokenContextDiskDir("Error processing context.sh: %s" % (e)) except IOError as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) diff --git a/cloudinit/sources/DataSourceRbxCloud.py b/cloudinit/sources/DataSourceRbxCloud.py index 084cb7d5..38fb5421 100644 --- a/cloudinit/sources/DataSourceRbxCloud.py +++ b/cloudinit/sources/DataSourceRbxCloud.py @@ -15,6 +15,7 @@ import os.path from cloudinit import log as logging from cloudinit import sources +from cloudinit import subp from cloudinit import util from cloudinit.event import EventType @@ -43,11 +44,11 @@ def int2ip(addr): def _sub_arp(cmd): """ - Uses the prefered cloud-init subprocess def of util.subp + Uses the prefered cloud-init subprocess def of subp.subp and runs arping. Breaking this to a separate function for later use in mocking and unittests """ - return util.subp(['arping'] + cmd) + return subp.subp(['arping'] + cmd) def gratuitous_arp(items, distro): @@ -61,7 +62,7 @@ def gratuitous_arp(items, distro): source_param, item['source'], item['destination'] ]) - except util.ProcessExecutionError as error: + except subp.ProcessExecutionError as error: # warning, because the system is able to function properly # despite no success - some ARP table may be waiting for # expiration, but the system may continue diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index cf676504..843b3a2a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -33,6 +33,7 @@ import socket from cloudinit import log as logging from cloudinit import serial from cloudinit import sources +from cloudinit import subp from cloudinit import util from cloudinit.event import EventType @@ -696,9 +697,9 @@ def identify_file(content_f): cmd = ["file", "--brief", "--mime-type", content_f] f_type = None try: - (f_type, _err) = util.subp(cmd) + (f_type, _err) = subp.subp(cmd) LOG.debug("script %s mime type is %s", content_f, f_type) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: util.logexc( LOG, ("Failed to identify script type for %s" % content_f, e)) return None if f_type is None else f_type.strip() diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 82b6730c..7bace8ca 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -15,6 +15,7 @@ from cloudinit import temp_utils from contextlib import contextmanager from xml.etree import ElementTree +from cloudinit import subp from cloudinit import url_helper from cloudinit import util from cloudinit import version @@ -92,7 +93,7 @@ def get_boot_telemetry(): raise RuntimeError("Failed to determine kernel start timestamp") try: - out, _ = util.subp(['/bin/systemctl', + out, _ = subp.subp(['/bin/systemctl', 'show', '-p', 'UserspaceTimestampMonotonic'], capture=True) @@ -105,7 +106,7 @@ def get_boot_telemetry(): "UserspaceTimestampMonotonic from systemd") user_start = kernel_start + (float(tsm) / 1000000) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: raise RuntimeError("Failed to get UserspaceTimestampMonotonic: %s" % e) except ValueError as e: @@ -114,7 +115,7 @@ def get_boot_telemetry(): % e) try: - out, _ = util.subp(['/bin/systemctl', 'show', + out, _ = subp.subp(['/bin/systemctl', 'show', 'cloud-init-local', '-p', 'InactiveExitTimestampMonotonic'], capture=True) @@ -126,7 +127,7 @@ def get_boot_telemetry(): "InactiveExitTimestampMonotonic from systemd") cloudinit_activation = kernel_start + (float(tsm) / 1000000) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: raise RuntimeError("Failed to get InactiveExitTimestampMonotonic: %s" % e) except ValueError as e: @@ -284,7 +285,7 @@ class OpenSSLManager(object): LOG.debug('Certificate already generated.') return with cd(self.tmpdir): - util.subp([ + subp.subp([ 'openssl', 'req', '-x509', '-nodes', '-subj', '/CN=LinuxTransport', '-days', '32768', '-newkey', 'rsa:2048', '-keyout', self.certificate_names['private_key'], @@ -301,14 +302,14 @@ class OpenSSLManager(object): @azure_ds_telemetry_reporter def _run_x509_action(action, cert): cmd = ['openssl', 'x509', '-noout', action] - result, _ = util.subp(cmd, data=cert) + result, _ = subp.subp(cmd, data=cert) return result @azure_ds_telemetry_reporter def _get_ssh_key_from_cert(self, certificate): pub_key = self._run_x509_action('-pubkey', certificate) keygen_cmd = ['ssh-keygen', '-i', '-m', 'PKCS8', '-f', '/dev/stdin'] - ssh_key, _ = util.subp(keygen_cmd, data=pub_key) + ssh_key, _ = subp.subp(keygen_cmd, data=pub_key) return ssh_key @azure_ds_telemetry_reporter @@ -341,7 +342,7 @@ class OpenSSLManager(object): certificates_content.encode('utf-8'), ] with cd(self.tmpdir): - out, _ = util.subp( + out, _ = subp.subp( 'openssl cms -decrypt -in /dev/stdin -inkey' ' {private_key} -recip {certificate} | openssl pkcs12 -nodes' ' -password pass:'.format(**self.certificate_names), diff --git a/cloudinit/sources/helpers/digitalocean.py b/cloudinit/sources/helpers/digitalocean.py index 0e7cccac..f5bbe46a 100644 --- a/cloudinit/sources/helpers/digitalocean.py +++ b/cloudinit/sources/helpers/digitalocean.py @@ -8,6 +8,7 @@ import random from cloudinit import log as logging from cloudinit import net as cloudnet from cloudinit import url_helper +from cloudinit import subp from cloudinit import util NIC_MAP = {'public': 'eth0', 'private': 'eth1'} @@ -36,14 +37,14 @@ def assign_ipv4_link_local(nic=None): ip_addr_cmd = ['ip', 'addr', 'add', addr, 'dev', nic] ip_link_cmd = ['ip', 'link', 'set', 'dev', nic, 'up'] - if not util.which('ip'): + if not subp.which('ip'): raise RuntimeError("No 'ip' command available to configure ip4LL " "address") try: - util.subp(ip_addr_cmd) + subp.subp(ip_addr_cmd) LOG.debug("assigned ip4LL address '%s' to '%s'", addr, nic) - util.subp(ip_link_cmd) + subp.subp(ip_link_cmd) LOG.debug("brought device '%s' up", nic) except Exception: util.logexc(LOG, "ip4LL address assignment of '%s' to '%s' failed." @@ -74,7 +75,7 @@ def del_ipv4_link_local(nic=None): ip_addr_cmd = ['ip', 'addr', 'flush', 'dev', nic] try: - util.subp(ip_addr_cmd) + subp.subp(ip_addr_cmd) LOG.debug("removed ip4LL addresses from %s", nic) except Exception as e: diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index a4373f24..c538720a 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -16,6 +16,7 @@ from cloudinit import ec2_utils from cloudinit import log as logging from cloudinit import net from cloudinit import sources +from cloudinit import subp from cloudinit import url_helper from cloudinit import util from cloudinit.sources import BrokenMetadata @@ -110,7 +111,7 @@ class SourceMixin(object): dev_entries = util.find_devs_with(criteria) if dev_entries: device = dev_entries[0] - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass return device diff --git a/cloudinit/sources/helpers/vmware/imc/config_custom_script.py b/cloudinit/sources/helpers/vmware/imc/config_custom_script.py index 9f14770e..2ab22de9 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_custom_script.py +++ b/cloudinit/sources/helpers/vmware/imc/config_custom_script.py @@ -9,6 +9,7 @@ import logging import os import stat +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -61,7 +62,7 @@ class PreCustomScript(RunCustomScript): """Executing custom script with precustomization argument.""" LOG.debug("Executing pre-customization script") self.prepare_script() - util.subp([CustomScriptConstant.CUSTOM_SCRIPT, "precustomization"]) + subp.subp([CustomScriptConstant.CUSTOM_SCRIPT, "precustomization"]) class PostCustomScript(RunCustomScript): diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 77cbf3b6..3745a262 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -10,6 +10,7 @@ import os import re from cloudinit.net.network_state import mask_to_net_prefix +from cloudinit import subp from cloudinit import util logger = logging.getLogger(__name__) @@ -73,7 +74,7 @@ class NicConfigurator(object): The mac address(es) are in the lower case """ cmd = ['ip', 'addr', 'show'] - output, _err = util.subp(cmd) + output, _err = subp.subp(cmd) sections = re.split(r'\n\d+: ', '\n' + output)[1:] macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))' @@ -248,8 +249,8 @@ class NicConfigurator(object): logger.info('Clearing DHCP leases') # Ignore the return code 1. - util.subp(["pkill", "dhclient"], rcs=[0, 1]) - util.subp(["rm", "-f", "/var/lib/dhcp/*"]) + subp.subp(["pkill", "dhclient"], rcs=[0, 1]) + subp.subp(["rm", "-f", "/var/lib/dhcp/*"]) def configure(self, osfamily=None): """ diff --git a/cloudinit/sources/helpers/vmware/imc/config_passwd.py b/cloudinit/sources/helpers/vmware/imc/config_passwd.py index 8c91fa41..d16a7690 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_passwd.py +++ b/cloudinit/sources/helpers/vmware/imc/config_passwd.py @@ -9,6 +9,7 @@ import logging import os +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -56,10 +57,10 @@ class PasswordConfigurator(object): LOG.info('Expiring password.') for user in uidUserList: try: - util.subp(['passwd', '--expire', user]) - except util.ProcessExecutionError as e: + subp.subp(['passwd', '--expire', user]) + except subp.ProcessExecutionError as e: if os.path.exists('/usr/bin/chage'): - util.subp(['chage', '-d', '0', user]) + subp.subp(['chage', '-d', '0', user]) else: LOG.warning('Failed to expire password for %s with error: ' '%s', user, e) diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index c60a38d7..893b1365 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -10,7 +10,7 @@ import os import re import time -from cloudinit import util +from cloudinit import subp from .guestcust_event import GuestCustEventEnum from .guestcust_state import GuestCustStateEnum @@ -34,7 +34,7 @@ def send_rpc(rpc): try: logger.debug("Sending RPC command: %s", rpc) - (out, err) = util.subp(["vmware-rpctool", rpc], rcs=[0]) + (out, err) = subp.subp(["vmware-rpctool", rpc], rcs=[0]) # Remove the trailing newline in the output. if out: out = out.rstrip() @@ -128,7 +128,7 @@ def get_tools_config(section, key, defaultVal): not installed. """ - if not util.which('vmware-toolbox-cmd'): + if not subp.which('vmware-toolbox-cmd'): logger.debug( 'vmware-toolbox-cmd not installed, returning default value') return defaultVal @@ -137,7 +137,7 @@ def get_tools_config(section, key, defaultVal): cmd = ['vmware-toolbox-cmd', 'config', 'get', section, key] try: - (outText, _) = util.subp(cmd) + (outText, _) = subp.subp(cmd) m = re.match(r'([^=]+)=(.*)', outText) if m: retValue = m.group(2).strip() @@ -147,7 +147,7 @@ def get_tools_config(section, key, defaultVal): logger.debug( "Tools config: [%s] %s is not found, return default value: %s", section, key, retValue) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: logger.error("Failed running %s[%s]", cmd, e.exit_code) logger.exception(e) diff --git a/cloudinit/subp.py b/cloudinit/subp.py index 0ad09306..f8400b1f 100644 --- a/cloudinit/subp.py +++ b/cloudinit/subp.py @@ -1,9 +1,11 @@ # This file is part of cloud-init. See LICENSE file for license information. """Common utility functions for interacting with subprocess.""" -# TODO move subp shellify and runparts related functions out of util.py - import logging +import os +import subprocess + +from errno import ENOEXEC LOG = logging.getLogger(__name__) @@ -54,4 +56,299 @@ def prepend_base_command(base_command, commands): return fixed_commands +class ProcessExecutionError(IOError): + + MESSAGE_TMPL = ('%(description)s\n' + 'Command: %(cmd)s\n' + 'Exit code: %(exit_code)s\n' + 'Reason: %(reason)s\n' + 'Stdout: %(stdout)s\n' + 'Stderr: %(stderr)s') + empty_attr = '-' + + def __init__(self, stdout=None, stderr=None, + exit_code=None, cmd=None, + description=None, reason=None, + errno=None): + if not cmd: + self.cmd = self.empty_attr + else: + self.cmd = cmd + + if not description: + if not exit_code and errno == ENOEXEC: + self.description = 'Exec format error. Missing #! in script?' + else: + self.description = 'Unexpected error while running command.' + else: + self.description = description + + if not isinstance(exit_code, int): + self.exit_code = self.empty_attr + else: + self.exit_code = exit_code + + if not stderr: + if stderr is None: + self.stderr = self.empty_attr + else: + self.stderr = stderr + else: + self.stderr = self._indent_text(stderr) + + if not stdout: + if stdout is None: + self.stdout = self.empty_attr + else: + self.stdout = stdout + else: + self.stdout = self._indent_text(stdout) + + if reason: + self.reason = reason + else: + self.reason = self.empty_attr + + self.errno = errno + message = self.MESSAGE_TMPL % { + 'description': self._ensure_string(self.description), + 'cmd': self._ensure_string(self.cmd), + 'exit_code': self._ensure_string(self.exit_code), + 'stdout': self._ensure_string(self.stdout), + 'stderr': self._ensure_string(self.stderr), + 'reason': self._ensure_string(self.reason), + } + IOError.__init__(self, message) + + def _ensure_string(self, text): + """ + if data is bytes object, decode + """ + return text.decode() if isinstance(text, bytes) else text + + def _indent_text(self, text, indent_level=8): + """ + indent text on all but the first line, allowing for easy to read output + """ + cr = '\n' + indent = ' ' * indent_level + # if input is bytes, return bytes + if isinstance(text, bytes): + cr = cr.encode() + indent = indent.encode() + # remove any newlines at end of text first to prevent unneeded blank + # line in output + return text.rstrip(cr).replace(cr, cr + indent) + + +def subp(args, data=None, rcs=None, env=None, capture=True, + combine_capture=False, shell=False, + logstring=False, decode="replace", target=None, update_env=None, + status_cb=None): + """Run a subprocess. + + :param args: command to run in a list. [cmd, arg1, arg2...] + :param data: input to the command, made available on its stdin. + :param rcs: + a list of allowed return codes. If subprocess exits with a value not + in this list, a ProcessExecutionError will be raised. By default, + data is returned as a string. See 'decode' parameter. + :param env: a dictionary for the command's environment. + :param capture: + boolean indicating if output should be captured. If True, then stderr + and stdout will be returned. If False, they will not be redirected. + :param combine_capture: + boolean indicating if stderr should be redirected to stdout. When True, + interleaved stderr and stdout will be returned as the first element of + a tuple, the second will be empty string or bytes (per decode). + if combine_capture is True, then output is captured independent of + the value of capture. + :param shell: boolean indicating if this should be run with a shell. + :param logstring: + the command will be logged to DEBUG. If it contains info that should + not be logged, then logstring will be logged instead. + :param decode: + if False, no decoding will be done and returned stdout and stderr will + be bytes. Other allowed values are 'strict', 'ignore', and 'replace'. + These values are passed through to bytes().decode() as the 'errors' + parameter. There is no support for decoding to other than utf-8. + :param target: + not supported, kwarg present only to make function signature similar + to curtin's subp. + :param update_env: + update the enviornment for this command with this dictionary. + this will not affect the current processes os.environ. + :param status_cb: + call this fuction with a single string argument before starting + and after finishing. + + :return + if not capturing, return is (None, None) + if capturing, stdout and stderr are returned. + if decode: + entries in tuple will be python2 unicode or python3 string + if not decode: + entries in tuple will be python2 string or python3 bytes + """ + + # not supported in cloud-init (yet), for now kept in the call signature + # to ease maintaining code shared between cloud-init and curtin + if target is not None: + raise ValueError("target arg not supported by cloud-init") + + if rcs is None: + rcs = [0] + + devnull_fp = None + + if update_env: + if env is None: + env = os.environ + env = env.copy() + env.update(update_env) + + if target_path(target) != "/": + args = ['chroot', target] + list(args) + + if status_cb: + command = ' '.join(args) if isinstance(args, list) else args + status_cb('Begin run command: {command}\n'.format(command=command)) + if not logstring: + LOG.debug(("Running command %s with allowed return codes %s" + " (shell=%s, capture=%s)"), + args, rcs, shell, 'combine' if combine_capture else capture) + else: + LOG.debug(("Running hidden command to protect sensitive " + "input/output logstring: %s"), logstring) + + stdin = None + stdout = None + stderr = None + if capture: + stdout = subprocess.PIPE + stderr = subprocess.PIPE + if combine_capture: + stdout = subprocess.PIPE + stderr = subprocess.STDOUT + if data is None: + # using devnull assures any reads get null, rather + # than possibly waiting on input. + devnull_fp = open(os.devnull) + stdin = devnull_fp + else: + stdin = subprocess.PIPE + if not isinstance(data, bytes): + data = data.encode() + + # Popen converts entries in the arguments array from non-bytes to bytes. + # When locale is unset it may use ascii for that encoding which can + # cause UnicodeDecodeErrors. (LP: #1751051) + if isinstance(args, bytes): + bytes_args = args + elif isinstance(args, str): + bytes_args = args.encode("utf-8") + else: + bytes_args = [ + x if isinstance(x, bytes) else x.encode("utf-8") + for x in args] + try: + sp = subprocess.Popen(bytes_args, stdout=stdout, + stderr=stderr, stdin=stdin, + env=env, shell=shell) + (out, err) = sp.communicate(data) + except OSError as e: + if status_cb: + status_cb('ERROR: End run command: invalid command provided\n') + raise ProcessExecutionError( + cmd=args, reason=e, errno=e.errno, + stdout="-" if decode else b"-", + stderr="-" if decode else b"-") + finally: + if devnull_fp: + devnull_fp.close() + + # Just ensure blank instead of none. + if capture or combine_capture: + if not out: + out = b'' + if not err: + err = b'' + if decode: + def ldecode(data, m='utf-8'): + if not isinstance(data, bytes): + return data + return data.decode(m, decode) + + out = ldecode(out) + err = ldecode(err) + + rc = sp.returncode + if rc not in rcs: + if status_cb: + status_cb( + 'ERROR: End run command: exit({code})\n'.format(code=rc)) + raise ProcessExecutionError(stdout=out, stderr=err, + exit_code=rc, + cmd=args) + if status_cb: + status_cb('End run command: exit({code})\n'.format(code=rc)) + return (out, err) + + +def target_path(target, path=None): + # return 'path' inside target, accepting target as None + if target in (None, ""): + target = "/" + elif not isinstance(target, str): + raise ValueError("Unexpected input for target: %s" % target) + else: + target = os.path.abspath(target) + # abspath("//") returns "//" specifically for 2 slashes. + if target.startswith("//"): + target = target[1:] + + if not path: + return target + + # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /. + while len(path) and path[0] == "/": + path = path[1:] + + return os.path.join(target, path) + + +def which(program, search=None, target=None): + target = target_path(target) + + if os.path.sep in program: + # if program had a '/' in it, then do not search PATH + # 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls + # so effectively we set cwd to / (or target) + if is_exe(target_path(target, program)): + return program + + if search is None: + paths = [p.strip('"') for p in + os.environ.get("PATH", "").split(os.pathsep)] + if target == "/": + search = paths + else: + search = [p for p in paths if p.startswith("/")] + + # normalize path input + search = [os.path.abspath(p) for p in search] + + for path in search: + ppath = os.path.sep.join((path, program)) + if is_exe(target_path(target, ppath)): + return ppath + + return None + + +def is_exe(fpath): + # return boolean indicating if fpath exists and is executable. + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + # vi: ts=4 expandtab diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index f3ab7e8c..58f63b69 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -23,9 +23,10 @@ from cloudinit import distros from cloudinit import helpers as ch from cloudinit.sources import DataSourceNone from cloudinit.templater import JINJA_AVAILABLE +from cloudinit import subp from cloudinit import util -_real_subp = util.subp +_real_subp = subp.subp # Used for skipping tests SkipTest = unittest.SkipTest @@ -134,9 +135,9 @@ class CiTestCase(TestCase): self.old_handlers = self.logger.handlers self.logger.handlers = [handler] if self.allowed_subp is True: - util.subp = _real_subp + subp.subp = _real_subp else: - util.subp = self._fake_subp + subp.subp = self._fake_subp def _fake_subp(self, *args, **kwargs): if 'args' in kwargs: @@ -171,7 +172,7 @@ class CiTestCase(TestCase): # Remove the handler we setup logging.getLogger().handlers = self.old_handlers logging.getLogger().level = None - util.subp = _real_subp + subp.subp = _real_subp super(CiTestCase, self).tearDown() def tmp_dir(self, dir=None, cleanup=True): @@ -280,13 +281,13 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): mock.patch.object(mod, f, trap_func)) # Handle subprocess calls - func = getattr(util, 'subp') + func = getattr(subp, 'subp') def nsubp(*_args, **_kwargs): return ('', '') self.patched_funcs.enter_context( - mock.patch.object(util, 'subp', nsubp)) + mock.patch.object(subp, 'subp', nsubp)) def null_func(*_args, **_kwargs): return None diff --git a/cloudinit/tests/test_conftest.py b/cloudinit/tests/test_conftest.py index 773ef8fe..a6537248 100644 --- a/cloudinit/tests/test_conftest.py +++ b/cloudinit/tests/test_conftest.py @@ -1,6 +1,6 @@ import pytest -from cloudinit import util +from cloudinit import subp from cloudinit.tests.helpers import CiTestCase @@ -9,17 +9,17 @@ class TestDisableSubpUsage: def test_using_subp_raises_assertion_error(self): with pytest.raises(AssertionError): - util.subp(["some", "args"]) + subp.subp(["some", "args"]) def test_typeerrors_on_incorrect_usage(self): with pytest.raises(TypeError): # We are intentionally passing no value for a parameter, so: # pylint: disable=no-value-for-parameter - util.subp() + subp.subp() @pytest.mark.parametrize('disable_subp_usage', [False], indirect=True) def test_subp_usage_can_be_reenabled(self): - util.subp(['whoami']) + subp.subp(['whoami']) @pytest.mark.parametrize( 'disable_subp_usage', [['whoami'], 'whoami'], indirect=True) @@ -27,18 +27,18 @@ class TestDisableSubpUsage: # The two parameters test each potential invocation with a single # argument with pytest.raises(AssertionError) as excinfo: - util.subp(["some", "args"]) + subp.subp(["some", "args"]) assert "allowed: whoami" in str(excinfo.value) - util.subp(['whoami']) + subp.subp(['whoami']) @pytest.mark.parametrize( 'disable_subp_usage', [['whoami', 'bash']], indirect=True) def test_subp_usage_can_be_conditionally_reenabled_for_multiple_cmds(self): with pytest.raises(AssertionError) as excinfo: - util.subp(["some", "args"]) + subp.subp(["some", "args"]) assert "allowed: whoami,bash" in str(excinfo.value) - util.subp(['bash', '-c', 'true']) - util.subp(['whoami']) + subp.subp(['bash', '-c', 'true']) + subp.subp(['whoami']) class TestDisableSubpUsageInTestSubclass(CiTestCase): @@ -46,16 +46,16 @@ class TestDisableSubpUsageInTestSubclass(CiTestCase): def test_using_subp_raises_exception(self): with pytest.raises(Exception): - util.subp(["some", "args"]) + subp.subp(["some", "args"]) def test_typeerrors_on_incorrect_usage(self): with pytest.raises(TypeError): - util.subp() + subp.subp() def test_subp_usage_can_be_reenabled(self): _old_allowed_subp = self.allow_subp self.allowed_subp = True try: - util.subp(['bash', '-c', 'true']) + subp.subp(['bash', '-c', 'true']) finally: self.allowed_subp = _old_allowed_subp diff --git a/cloudinit/tests/test_gpg.py b/cloudinit/tests/test_gpg.py index 8dd57137..f96f5372 100644 --- a/cloudinit/tests/test_gpg.py +++ b/cloudinit/tests/test_gpg.py @@ -4,19 +4,19 @@ from unittest import mock from cloudinit import gpg -from cloudinit import util +from cloudinit import subp from cloudinit.tests.helpers import CiTestCase @mock.patch("cloudinit.gpg.time.sleep") -@mock.patch("cloudinit.gpg.util.subp") +@mock.patch("cloudinit.gpg.subp.subp") class TestReceiveKeys(CiTestCase): """Test the recv_key method.""" def test_retries_on_subp_exc(self, m_subp, m_sleep): """retry should be done on gpg receive keys failure.""" retries = (1, 2, 4) - my_exc = util.ProcessExecutionError( + my_exc = subp.ProcessExecutionError( stdout='', stderr='', exit_code=2, cmd=['mycmd']) m_subp.side_effect = (my_exc, my_exc, ('', '')) gpg.recv_key("ABCD", "keyserver.example.com", retries=retries) @@ -26,7 +26,7 @@ class TestReceiveKeys(CiTestCase): """If the final run fails, error should be raised.""" naplen = 1 keyid, keyserver = ("ABCD", "keyserver.example.com") - m_subp.side_effect = util.ProcessExecutionError( + m_subp.side_effect = subp.ProcessExecutionError( stdout='', stderr='', exit_code=2, cmd=['mycmd']) with self.assertRaises(ValueError) as rcm: gpg.recv_key(keyid, keyserver, retries=(naplen,)) @@ -36,7 +36,7 @@ class TestReceiveKeys(CiTestCase): def test_no_retries_on_none(self, m_subp, m_sleep): """retry should not be done if retries is None.""" - m_subp.side_effect = util.ProcessExecutionError( + m_subp.side_effect = subp.ProcessExecutionError( stdout='', stderr='', exit_code=2, cmd=['mycmd']) with self.assertRaises(ValueError): gpg.recv_key("ABCD", "keyserver.example.com", retries=None) diff --git a/cloudinit/tests/test_netinfo.py b/cloudinit/tests/test_netinfo.py index 1c8a791e..e44b16d8 100644 --- a/cloudinit/tests/test_netinfo.py +++ b/cloudinit/tests/test_netinfo.py @@ -27,8 +27,8 @@ class TestNetInfo(CiTestCase): maxDiff = None with_logs = True - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_old_nettools_pformat(self, m_subp, m_which): """netdev_pformat properly rendering old nettools info.""" m_subp.return_value = (SAMPLE_OLD_IFCONFIG_OUT, '') @@ -36,8 +36,8 @@ class TestNetInfo(CiTestCase): content = netdev_pformat() self.assertEqual(NETDEV_FORMATTED_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_new_nettools_pformat(self, m_subp, m_which): """netdev_pformat properly rendering netdev new nettools info.""" m_subp.return_value = (SAMPLE_NEW_IFCONFIG_OUT, '') @@ -45,8 +45,8 @@ class TestNetInfo(CiTestCase): content = netdev_pformat() self.assertEqual(NETDEV_FORMATTED_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_freebsd_nettools_pformat(self, m_subp, m_which): """netdev_pformat properly rendering netdev new nettools info.""" m_subp.return_value = (SAMPLE_FREEBSD_IFCONFIG_OUT, '') @@ -57,8 +57,8 @@ class TestNetInfo(CiTestCase): print() self.assertEqual(FREEBSD_NETDEV_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_iproute_pformat(self, m_subp, m_which): """netdev_pformat properly rendering ip route info.""" m_subp.return_value = (SAMPLE_IPADDRSHOW_OUT, '') @@ -72,8 +72,8 @@ class TestNetInfo(CiTestCase): '255.0.0.0 | . |', '255.0.0.0 | host |') self.assertEqual(new_output, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_warn_on_missing_commands(self, m_subp, m_which): """netdev_pformat warns when missing both ip and 'netstat'.""" m_which.return_value = None # Niether ip nor netstat found @@ -85,8 +85,8 @@ class TestNetInfo(CiTestCase): self.logs.getvalue()) m_subp.assert_not_called() - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_info_nettools_down(self, m_subp, m_which): """test netdev_info using nettools and down interfaces.""" m_subp.return_value = ( @@ -100,8 +100,8 @@ class TestNetInfo(CiTestCase): 'hwaddr': '.', 'up': True}}, netdev_info(".")) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_info_iproute_down(self, m_subp, m_which): """Test netdev_info with ip and down interfaces.""" m_subp.return_value = ( @@ -130,8 +130,8 @@ class TestNetInfo(CiTestCase): readResource("netinfo/netdev-formatted-output-down"), netdev_pformat()) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_route_nettools_pformat(self, m_subp, m_which): """route_pformat properly rendering nettools route info.""" @@ -147,8 +147,8 @@ class TestNetInfo(CiTestCase): content = route_pformat() self.assertEqual(ROUTE_FORMATTED_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_route_iproute_pformat(self, m_subp, m_which): """route_pformat properly rendering ip route info.""" @@ -165,8 +165,8 @@ class TestNetInfo(CiTestCase): content = route_pformat() self.assertEqual(ROUTE_FORMATTED_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_route_warn_on_missing_commands(self, m_subp, m_which): """route_pformat warns when missing both ip and 'netstat'.""" m_which.return_value = None # Niether ip nor netstat found diff --git a/cloudinit/tests/test_subp.py b/cloudinit/tests/test_subp.py index 448097d3..911c1f3d 100644 --- a/cloudinit/tests/test_subp.py +++ b/cloudinit/tests/test_subp.py @@ -2,10 +2,21 @@ """Tests for cloudinit.subp utility functions""" -from cloudinit import subp +import json +import os +import sys +import stat + +from unittest import mock + +from cloudinit import subp, util from cloudinit.tests.helpers import CiTestCase +BASH = subp.which('bash') +BOGUS_COMMAND = 'this-is-not-expected-to-be-a-program-name' + + class TestPrependBaseCommands(CiTestCase): with_logs = True @@ -58,4 +69,218 @@ class TestPrependBaseCommands(CiTestCase): self.assertEqual('', self.logs.getvalue()) self.assertEqual(expected, fixed_commands) + +class TestSubp(CiTestCase): + allowed_subp = [BASH, 'cat', CiTestCase.SUBP_SHELL_TRUE, + BOGUS_COMMAND, sys.executable] + + stdin2err = [BASH, '-c', 'cat >&2'] + stdin2out = ['cat'] + utf8_invalid = b'ab\xaadef' + utf8_valid = b'start \xc3\xa9 end' + utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7' + printenv = [BASH, '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--'] + + def printf_cmd(self, *args): + # bash's printf supports \xaa. So does /usr/bin/printf + # but by using bash, we remove dependency on another program. + return([BASH, '-c', 'printf "$@"', 'printf'] + list(args)) + + def test_subp_handles_bytestrings(self): + """subp can run a bytestring command if shell is True.""" + tmp_file = self.tmp_path('test.out') + cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file) + (out, _err) = subp.subp(cmd.encode('utf-8'), shell=True) + self.assertEqual(u'', out) + self.assertEqual(u'', _err) + self.assertEqual('HI MOM\n', util.load_file(tmp_file)) + + def test_subp_handles_strings(self): + """subp can run a string command if shell is True.""" + tmp_file = self.tmp_path('test.out') + cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file) + (out, _err) = subp.subp(cmd, shell=True) + self.assertEqual(u'', out) + self.assertEqual(u'', _err) + self.assertEqual('HI MOM\n', util.load_file(tmp_file)) + + def test_subp_handles_utf8(self): + # The given bytes contain utf-8 accented characters as seen in e.g. + # the "deja dup" package in Ubuntu. + cmd = self.printf_cmd(self.utf8_valid_2) + (out, _err) = subp.subp(cmd, capture=True) + self.assertEqual(out, self.utf8_valid_2.decode('utf-8')) + + def test_subp_respects_decode_false(self): + (out, err) = subp.subp(self.stdin2out, capture=True, decode=False, + data=self.utf8_valid) + self.assertTrue(isinstance(out, bytes)) + self.assertTrue(isinstance(err, bytes)) + self.assertEqual(out, self.utf8_valid) + + def test_subp_decode_ignore(self): + # this executes a string that writes invalid utf-8 to stdout + (out, _err) = subp.subp(self.printf_cmd('abc\\xaadef'), + capture=True, decode='ignore') + self.assertEqual(out, 'abcdef') + + def test_subp_decode_strict_valid_utf8(self): + (out, _err) = subp.subp(self.stdin2out, capture=True, + decode='strict', data=self.utf8_valid) + self.assertEqual(out, self.utf8_valid.decode('utf-8')) + + def test_subp_decode_invalid_utf8_replaces(self): + (out, _err) = subp.subp(self.stdin2out, capture=True, + data=self.utf8_invalid) + expected = self.utf8_invalid.decode('utf-8', 'replace') + self.assertEqual(out, expected) + + def test_subp_decode_strict_raises(self): + args = [] + kwargs = {'args': self.stdin2out, 'capture': True, + 'decode': 'strict', 'data': self.utf8_invalid} + self.assertRaises(UnicodeDecodeError, subp.subp, *args, **kwargs) + + def test_subp_capture_stderr(self): + data = b'hello world' + (out, err) = subp.subp(self.stdin2err, capture=True, + decode=False, data=data, + update_env={'LC_ALL': 'C'}) + self.assertEqual(err, data) + self.assertEqual(out, b'') + + def test_subp_reads_env(self): + with mock.patch.dict("os.environ", values={'FOO': 'BAR'}): + out, _err = subp.subp(self.printenv + ['FOO'], capture=True) + self.assertEqual('FOO=BAR', out.splitlines()[0]) + + def test_subp_env_and_update_env(self): + out, _err = subp.subp( + self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True, + env={'FOO': 'BAR'}, + update_env={'HOME': '/myhome', 'K2': 'V2'}) + self.assertEqual( + ['FOO=BAR', 'HOME=/myhome', 'K1=', 'K2=V2'], out.splitlines()) + + def test_subp_update_env(self): + extra = {'FOO': 'BAR', 'HOME': '/root', 'K1': 'V1'} + with mock.patch.dict("os.environ", values=extra): + out, _err = subp.subp( + self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True, + update_env={'HOME': '/myhome', 'K2': 'V2'}) + + self.assertEqual( + ['FOO=BAR', 'HOME=/myhome', 'K1=V1', 'K2=V2'], out.splitlines()) + + def test_subp_warn_missing_shebang(self): + """Warn on no #! in script""" + noshebang = self.tmp_path('noshebang') + util.write_file(noshebang, 'true\n') + + print("os is %s" % os) + os.chmod(noshebang, os.stat(noshebang).st_mode | stat.S_IEXEC) + with self.allow_subp([noshebang]): + self.assertRaisesRegex(subp.ProcessExecutionError, + r'Missing #! in script\?', + subp.subp, (noshebang,)) + + def test_subp_combined_stderr_stdout(self): + """Providing combine_capture as True redirects stderr to stdout.""" + data = b'hello world' + (out, err) = subp.subp(self.stdin2err, capture=True, + combine_capture=True, decode=False, data=data) + self.assertEqual(b'', err) + self.assertEqual(data, out) + + def test_returns_none_if_no_capture(self): + (out, err) = subp.subp(self.stdin2out, data=b'', capture=False) + self.assertIsNone(err) + self.assertIsNone(out) + + def test_exception_has_out_err_are_bytes_if_decode_false(self): + """Raised exc should have stderr, stdout as bytes if no decode.""" + with self.assertRaises(subp.ProcessExecutionError) as cm: + subp.subp([BOGUS_COMMAND], decode=False) + self.assertTrue(isinstance(cm.exception.stdout, bytes)) + self.assertTrue(isinstance(cm.exception.stderr, bytes)) + + def test_exception_has_out_err_are_bytes_if_decode_true(self): + """Raised exc should have stderr, stdout as string if no decode.""" + with self.assertRaises(subp.ProcessExecutionError) as cm: + subp.subp([BOGUS_COMMAND], decode=True) + self.assertTrue(isinstance(cm.exception.stdout, str)) + self.assertTrue(isinstance(cm.exception.stderr, str)) + + def test_bunch_of_slashes_in_path(self): + self.assertEqual("/target/my/path/", + subp.target_path("/target/", "//my/path/")) + self.assertEqual("/target/my/path/", + subp.target_path("/target/", "///my/path/")) + + def test_c_lang_can_take_utf8_args(self): + """Independent of system LC_CTYPE, args can contain utf-8 strings. + + When python starts up, its default encoding gets set based on + the value of LC_CTYPE. If no system locale is set, the default + encoding for both python2 and python3 in some paths will end up + being ascii. + + Attempts to use setlocale or patching (or changing) os.environ + in the current environment seem to not be effective. + + This test starts up a python with LC_CTYPE set to C so that + the default encoding will be set to ascii. In such an environment + Popen(['command', 'non-ascii-arg']) would cause a UnicodeDecodeError. + """ + python_prog = '\n'.join([ + 'import json, sys', + 'from cloudinit.subp import subp', + 'data = sys.stdin.read()', + 'cmd = json.loads(data)', + 'subp(cmd, capture=False)', + '']) + cmd = [BASH, '-c', 'echo -n "$@"', '--', + self.utf8_valid.decode("utf-8")] + python_subp = [sys.executable, '-c', python_prog] + + out, _err = subp.subp( + python_subp, update_env={'LC_CTYPE': 'C'}, + data=json.dumps(cmd).encode("utf-8"), + decode=False) + self.assertEqual(self.utf8_valid, out) + + def test_bogus_command_logs_status_messages(self): + """status_cb gets status messages logs on bogus commands provided.""" + logs = [] + + def status_cb(log): + logs.append(log) + + with self.assertRaises(subp.ProcessExecutionError): + subp.subp([BOGUS_COMMAND], status_cb=status_cb) + + expected = [ + 'Begin run command: {cmd}\n'.format(cmd=BOGUS_COMMAND), + 'ERROR: End run command: invalid command provided\n'] + self.assertEqual(expected, logs) + + def test_command_logs_exit_codes_to_status_cb(self): + """status_cb gets status messages containing command exit code.""" + logs = [] + + def status_cb(log): + logs.append(log) + + with self.assertRaises(subp.ProcessExecutionError): + subp.subp([BASH, '-c', 'exit 2'], status_cb=status_cb) + subp.subp([BASH, '-c', 'exit 0'], status_cb=status_cb) + + expected = [ + 'Begin run command: %s -c exit 2\n' % BASH, + 'ERROR: End run command: exit(2)\n', + 'Begin run command: %s -c exit 0\n' % BASH, + 'End run command: exit(0)\n'] + self.assertEqual(expected, logs) + + # vi: ts=4 expandtab diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index bfccfe1e..0d9a8fd9 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -9,6 +9,7 @@ import platform import pytest import cloudinit.util as util +from cloudinit import subp from cloudinit.tests.helpers import CiTestCase, mock from textwrap import dedent @@ -332,7 +333,7 @@ class TestBlkid(CiTestCase): "PARTUUID": self.ids["id09"]}, }) - @mock.patch("cloudinit.util.subp") + @mock.patch("cloudinit.subp.subp") def test_functional_blkid(self, m_subp): m_subp.return_value = ( self.blkid_out.format(**self.ids), "") @@ -340,7 +341,7 @@ class TestBlkid(CiTestCase): m_subp.assert_called_with(["blkid", "-o", "full"], capture=True, decode="replace") - @mock.patch("cloudinit.util.subp") + @mock.patch("cloudinit.subp.subp") def test_blkid_no_cache_uses_no_cache(self, m_subp): """blkid should turn off cache if disable_cache is true.""" m_subp.return_value = ( @@ -351,7 +352,7 @@ class TestBlkid(CiTestCase): capture=True, decode="replace") -@mock.patch('cloudinit.util.subp') +@mock.patch('cloudinit.subp.subp') class TestUdevadmSettle(CiTestCase): def test_with_no_params(self, m_subp): """called with no parameters.""" @@ -396,8 +397,8 @@ class TestUdevadmSettle(CiTestCase): '--timeout=%s' % timeout]) def test_subp_exception_raises_to_caller(self, m_subp): - m_subp.side_effect = util.ProcessExecutionError("BOOM") - self.assertRaises(util.ProcessExecutionError, util.udevadm_settle) + m_subp.side_effect = subp.ProcessExecutionError("BOOM") + self.assertRaises(subp.ProcessExecutionError, util.udevadm_settle) @mock.patch('os.path.exists') diff --git a/cloudinit/util.py b/cloudinit/util.py index 985e7d20..445e3d4c 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -32,12 +32,13 @@ import subprocess import sys import time from base64 import b64decode, b64encode -from errno import ENOENT, ENOEXEC +from errno import ENOENT from functools import lru_cache from urllib import parse from cloudinit import importer from cloudinit import log as logging +from cloudinit import subp from cloudinit import ( mergers, safeyaml, @@ -74,8 +75,8 @@ def get_dpkg_architecture(target=None): N.B. This function is wrapped in functools.lru_cache, so repeated calls won't shell out every time. """ - out, _ = subp(['dpkg', '--print-architecture'], capture=True, - target=target) + out, _ = subp.subp(['dpkg', '--print-architecture'], capture=True, + target=target) return out.strip() @@ -86,7 +87,8 @@ def lsb_release(target=None): data = {} try: - out, _ = subp(['lsb_release', '--all'], capture=True, target=target) + out, _ = subp.subp(['lsb_release', '--all'], capture=True, + target=target) for line in out.splitlines(): fname, _, val = line.partition(":") if fname in fmap: @@ -96,35 +98,13 @@ def lsb_release(target=None): LOG.warning("Missing fields in lsb_release --all output: %s", ','.join(missing)) - except ProcessExecutionError as err: + except subp.ProcessExecutionError as err: LOG.warning("Unable to get lsb_release --all: %s", err) data = dict((v, "UNAVAILABLE") for v in fmap.values()) return data -def target_path(target, path=None): - # return 'path' inside target, accepting target as None - if target in (None, ""): - target = "/" - elif not isinstance(target, str): - raise ValueError("Unexpected input for target: %s" % target) - else: - target = os.path.abspath(target) - # abspath("//") returns "//" specifically for 2 slashes. - if target.startswith("//"): - target = target[1:] - - if not path: - return target - - # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /. - while len(path) and path[0] == "/": - path = path[1:] - - return os.path.join(target, path) - - def decode_binary(blob, encoding='utf-8'): # Converts a binary type into a text type using given encoding. if isinstance(blob, str): @@ -201,91 +181,6 @@ DMIDECODE_TO_DMI_SYS_MAPPING = { } -class ProcessExecutionError(IOError): - - MESSAGE_TMPL = ('%(description)s\n' - 'Command: %(cmd)s\n' - 'Exit code: %(exit_code)s\n' - 'Reason: %(reason)s\n' - 'Stdout: %(stdout)s\n' - 'Stderr: %(stderr)s') - empty_attr = '-' - - def __init__(self, stdout=None, stderr=None, - exit_code=None, cmd=None, - description=None, reason=None, - errno=None): - if not cmd: - self.cmd = self.empty_attr - else: - self.cmd = cmd - - if not description: - if not exit_code and errno == ENOEXEC: - self.description = 'Exec format error. Missing #! in script?' - else: - self.description = 'Unexpected error while running command.' - else: - self.description = description - - if not isinstance(exit_code, int): - self.exit_code = self.empty_attr - else: - self.exit_code = exit_code - - if not stderr: - if stderr is None: - self.stderr = self.empty_attr - else: - self.stderr = stderr - else: - self.stderr = self._indent_text(stderr) - - if not stdout: - if stdout is None: - self.stdout = self.empty_attr - else: - self.stdout = stdout - else: - self.stdout = self._indent_text(stdout) - - if reason: - self.reason = reason - else: - self.reason = self.empty_attr - - self.errno = errno - message = self.MESSAGE_TMPL % { - 'description': self._ensure_string(self.description), - 'cmd': self._ensure_string(self.cmd), - 'exit_code': self._ensure_string(self.exit_code), - 'stdout': self._ensure_string(self.stdout), - 'stderr': self._ensure_string(self.stderr), - 'reason': self._ensure_string(self.reason), - } - IOError.__init__(self, message) - - def _ensure_string(self, text): - """ - if data is bytes object, decode - """ - return text.decode() if isinstance(text, bytes) else text - - def _indent_text(self, text, indent_level=8): - """ - indent text on all but the first line, allowing for easy to read output - """ - cr = '\n' - indent = ' ' * indent_level - # if input is bytes, return bytes - if isinstance(text, bytes): - cr = cr.encode() - indent = indent.encode() - # remove any newlines at end of text first to prevent unneeded blank - # line in output - return text.rstrip(cr).replace(cr, cr + indent) - - class SeLinuxGuard(object): def __init__(self, path, recursive=False): # Late import since it might not always @@ -875,8 +770,8 @@ def runparts(dirp, skip_no_exist=True, exe_prefix=None): if os.path.isfile(exe_path) and os.access(exe_path, os.X_OK): attempted.append(exe_path) try: - subp(prefix + [exe_path], capture=False) - except ProcessExecutionError as e: + subp.subp(prefix + [exe_path], capture=False) + except subp.ProcessExecutionError as e: logexc(LOG, "Failed running %s [%s]", exe_path, e.exit_code) failed.append(e) @@ -1271,10 +1166,10 @@ def find_devs_with_netbsd(criteria=None, oformat='device', label = criteria.lstrip("LABEL=") if criteria.startswith("TYPE="): _type = criteria.lstrip("TYPE=") - out, _err = subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) + out, _err = subp.subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) for dev in out.split(): if label or _type: - mscdlabel_out, _ = subp(['mscdlabel', dev], rcs=[0, 1]) + mscdlabel_out, _ = subp.subp(['mscdlabel', dev], rcs=[0, 1]) if label and not ('label "%s"' % label) in mscdlabel_out: continue if _type == "iso9660" and "ISO filesystem" not in mscdlabel_out: @@ -1287,7 +1182,7 @@ def find_devs_with_netbsd(criteria=None, oformat='device', def find_devs_with_openbsd(criteria=None, oformat='device', tag=None, no_cache=False, path=None): - out, _err = subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) + out, _err = subp.subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) devlist = [] for entry in out.split(','): if not entry.endswith(':'): @@ -1353,8 +1248,8 @@ def find_devs_with(criteria=None, oformat='device', cmd = blk_id_cmd + options # See man blkid for why 2 is added try: - (out, _err) = subp(cmd, rcs=[0, 2]) - except ProcessExecutionError as e: + (out, _err) = subp.subp(cmd, rcs=[0, 2]) + except subp.ProcessExecutionError as e: if e.errno == ENOENT: # blkid not found... out = "" @@ -1389,7 +1284,7 @@ def blkid(devs=None, disable_cache=False): # we have to decode with 'replace' as shelx.split (called by # load_shell_content) can't take bytes. So this is potentially # lossy of non-utf-8 chars in blkid output. - out, _ = subp(cmd, capture=True, decode="replace") + out, _ = subp.subp(cmd, capture=True, decode="replace") ret = {} for line in out.splitlines(): dev, _, data = line.partition(":") @@ -1709,7 +1604,7 @@ def unmounter(umount): finally: if umount: umount_cmd = ["umount", umount] - subp(umount_cmd) + subp.subp(umount_cmd) def mounts(): @@ -1720,7 +1615,7 @@ def mounts(): mount_locs = load_file("/proc/mounts").splitlines() method = 'proc' else: - (mountoutput, _err) = subp("mount") + (mountoutput, _err) = subp.subp("mount") mount_locs = mountoutput.splitlines() method = 'mount' mountre = r'^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$' @@ -1804,7 +1699,7 @@ def mount_cb(device, callback, data=None, mtype=None, mountcmd.extend(['-t', mtype]) mountcmd.append(device) mountcmd.append(tmpd) - subp(mountcmd, update_env=update_env_for_mount) + subp.subp(mountcmd, update_env=update_env_for_mount) umount = tmpd # This forces it to be unmounted (when set) mountpoint = tmpd break @@ -1988,185 +1883,6 @@ def delete_dir_contents(dirname): del_file(node_fullpath) -def subp_blob_in_tempfile(blob, *args, **kwargs): - """Write blob to a tempfile, and call subp with args, kwargs. Then cleanup. - - 'basename' as a kwarg allows providing the basename for the file. - The 'args' argument to subp will be updated with the full path to the - filename as the first argument. - """ - basename = kwargs.pop('basename', "subp_blob") - - if len(args) == 0 and 'args' not in kwargs: - args = [tuple()] - - # Use tmpdir over tmpfile to avoid 'text file busy' on execute - with temp_utils.tempdir(needs_exe=True) as tmpd: - tmpf = os.path.join(tmpd, basename) - if 'args' in kwargs: - kwargs['args'] = [tmpf] + list(kwargs['args']) - else: - args = list(args) - args[0] = [tmpf] + args[0] - - write_file(tmpf, blob, mode=0o700) - return subp(*args, **kwargs) - - -def subp(args, data=None, rcs=None, env=None, capture=True, - combine_capture=False, shell=False, - logstring=False, decode="replace", target=None, update_env=None, - status_cb=None): - """Run a subprocess. - - :param args: command to run in a list. [cmd, arg1, arg2...] - :param data: input to the command, made available on its stdin. - :param rcs: - a list of allowed return codes. If subprocess exits with a value not - in this list, a ProcessExecutionError will be raised. By default, - data is returned as a string. See 'decode' parameter. - :param env: a dictionary for the command's environment. - :param capture: - boolean indicating if output should be captured. If True, then stderr - and stdout will be returned. If False, they will not be redirected. - :param combine_capture: - boolean indicating if stderr should be redirected to stdout. When True, - interleaved stderr and stdout will be returned as the first element of - a tuple, the second will be empty string or bytes (per decode). - if combine_capture is True, then output is captured independent of - the value of capture. - :param shell: boolean indicating if this should be run with a shell. - :param logstring: - the command will be logged to DEBUG. If it contains info that should - not be logged, then logstring will be logged instead. - :param decode: - if False, no decoding will be done and returned stdout and stderr will - be bytes. Other allowed values are 'strict', 'ignore', and 'replace'. - These values are passed through to bytes().decode() as the 'errors' - parameter. There is no support for decoding to other than utf-8. - :param target: - not supported, kwarg present only to make function signature similar - to curtin's subp. - :param update_env: - update the enviornment for this command with this dictionary. - this will not affect the current processes os.environ. - :param status_cb: - call this fuction with a single string argument before starting - and after finishing. - - :return - if not capturing, return is (None, None) - if capturing, stdout and stderr are returned. - if decode: - entries in tuple will be python2 unicode or python3 string - if not decode: - entries in tuple will be python2 string or python3 bytes - """ - - # not supported in cloud-init (yet), for now kept in the call signature - # to ease maintaining code shared between cloud-init and curtin - if target is not None: - raise ValueError("target arg not supported by cloud-init") - - if rcs is None: - rcs = [0] - - devnull_fp = None - - if update_env: - if env is None: - env = os.environ - env = env.copy() - env.update(update_env) - - if target_path(target) != "/": - args = ['chroot', target] + list(args) - - if status_cb: - command = ' '.join(args) if isinstance(args, list) else args - status_cb('Begin run command: {command}\n'.format(command=command)) - if not logstring: - LOG.debug(("Running command %s with allowed return codes %s" - " (shell=%s, capture=%s)"), - args, rcs, shell, 'combine' if combine_capture else capture) - else: - LOG.debug(("Running hidden command to protect sensitive " - "input/output logstring: %s"), logstring) - - stdin = None - stdout = None - stderr = None - if capture: - stdout = subprocess.PIPE - stderr = subprocess.PIPE - if combine_capture: - stdout = subprocess.PIPE - stderr = subprocess.STDOUT - if data is None: - # using devnull assures any reads get null, rather - # than possibly waiting on input. - devnull_fp = open(os.devnull) - stdin = devnull_fp - else: - stdin = subprocess.PIPE - if not isinstance(data, bytes): - data = data.encode() - - # Popen converts entries in the arguments array from non-bytes to bytes. - # When locale is unset it may use ascii for that encoding which can - # cause UnicodeDecodeErrors. (LP: #1751051) - if isinstance(args, bytes): - bytes_args = args - elif isinstance(args, str): - bytes_args = args.encode("utf-8") - else: - bytes_args = [ - x if isinstance(x, bytes) else x.encode("utf-8") - for x in args] - try: - sp = subprocess.Popen(bytes_args, stdout=stdout, - stderr=stderr, stdin=stdin, - env=env, shell=shell) - (out, err) = sp.communicate(data) - except OSError as e: - if status_cb: - status_cb('ERROR: End run command: invalid command provided\n') - raise ProcessExecutionError( - cmd=args, reason=e, errno=e.errno, - stdout="-" if decode else b"-", - stderr="-" if decode else b"-") - finally: - if devnull_fp: - devnull_fp.close() - - # Just ensure blank instead of none. - if capture or combine_capture: - if not out: - out = b'' - if not err: - err = b'' - if decode: - def ldecode(data, m='utf-8'): - if not isinstance(data, bytes): - return data - return data.decode(m, decode) - - out = ldecode(out) - err = ldecode(err) - - rc = sp.returncode - if rc not in rcs: - if status_cb: - status_cb( - 'ERROR: End run command: exit({code})\n'.format(code=rc)) - raise ProcessExecutionError(stdout=out, stderr=err, - exit_code=rc, - cmd=args) - if status_cb: - status_cb('End run command: exit({code})\n'.format(code=rc)) - return (out, err) - - def make_header(comment_char="#", base='created'): ci_ver = version.version_string() header = str(comment_char) @@ -2232,7 +1948,7 @@ def is_container(): try: # try to run a helper program. if it returns true/zero # then we're inside a container. otherwise, no - subp(helper) + subp.subp(helper) return True except (IOError, OSError): pass @@ -2438,7 +2154,7 @@ def find_freebsd_part(fs): return splitted[2] elif splitted[2] in ['label', 'gpt', 'ufs']: target_label = fs[5:] - (part, _err) = subp(['glabel', 'status', '-s']) + (part, _err) = subp.subp(['glabel', 'status', '-s']) for labels in part.split("\n"): items = labels.split() if len(items) > 0 and items[0] == target_label: @@ -2460,10 +2176,10 @@ def get_path_dev_freebsd(path, mnt_list): def get_mount_info_freebsd(path): - (result, err) = subp(['mount', '-p', path], rcs=[0, 1]) + (result, err) = subp.subp(['mount', '-p', path], rcs=[0, 1]) if len(err): # find a path if the input is not a mounting point - (mnt_list, err) = subp(['mount', '-p']) + (mnt_list, err) = subp.subp(['mount', '-p']) path_found = get_path_dev_freebsd(path, mnt_list) if (path_found is None): return None @@ -2479,8 +2195,8 @@ def get_device_info_from_zpool(zpool): LOG.debug('Cannot get zpool info, no /dev/zfs') return None try: - (zpoolstatus, err) = subp(['zpool', 'status', zpool]) - except ProcessExecutionError as err: + (zpoolstatus, err) = subp.subp(['zpool', 'status', zpool]) + except subp.ProcessExecutionError as err: LOG.warning("Unable to get zpool status of %s: %s", zpool, err) return None if len(err): @@ -2494,7 +2210,7 @@ def get_device_info_from_zpool(zpool): def parse_mount(path): - (mountoutput, _err) = subp(['mount']) + (mountoutput, _err) = subp.subp(['mount']) mount_locs = mountoutput.splitlines() # there are 2 types of mount outputs we have to parse therefore # the regex is a bit complex. to better understand this regex see: @@ -2567,40 +2283,6 @@ def get_mount_info(path, log=LOG, get_mnt_opts=False): return parse_mount(path) -def is_exe(fpath): - # return boolean indicating if fpath exists and is executable. - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - -def which(program, search=None, target=None): - target = target_path(target) - - if os.path.sep in program: - # if program had a '/' in it, then do not search PATH - # 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls - # so effectively we set cwd to / (or target) - if is_exe(target_path(target, program)): - return program - - if search is None: - paths = [p.strip('"') for p in - os.environ.get("PATH", "").split(os.pathsep)] - if target == "/": - search = paths - else: - search = [p for p in paths if p.startswith("/")] - - # normalize path input - search = [os.path.abspath(p) for p in search] - - for path in search: - ppath = os.path.sep.join((path, program)) - if is_exe(target_path(target, ppath)): - return ppath - - return None - - def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): if args is None: args = [] @@ -2764,7 +2446,7 @@ def _call_dmidecode(key, dmidecode_path): """ try: cmd = [dmidecode_path, "--string", key] - (result, _err) = subp(cmd) + (result, _err) = subp.subp(cmd) result = result.strip() LOG.debug("dmidecode returned '%s' for '%s'", result, key) if result.replace(".", "") == "": @@ -2818,7 +2500,8 @@ def read_dmi_data(key): LOG.debug("dmidata is not supported on %s", uname_arch) return None - dmidecode_path = which('dmidecode') + print("hi, now its: %s\n", subp) + dmidecode_path = subp.which('dmidecode') if dmidecode_path: return _call_dmidecode(key, dmidecode_path) @@ -2834,7 +2517,7 @@ def message_from_string(string): def get_installed_packages(target=None): - (out, _) = subp(['dpkg-query', '--list'], target=target, capture=True) + (out, _) = subp.subp(['dpkg-query', '--list'], target=target, capture=True) pkgs_inst = set() for line in out.splitlines(): @@ -2970,7 +2653,7 @@ def udevadm_settle(exists=None, timeout=None): if timeout: settle_cmd.extend(['--timeout=%s' % timeout]) - return subp(settle_cmd) + return subp.subp(settle_cmd) def get_proc_ppid(pid): diff --git a/packages/bddeb b/packages/bddeb index 02ac2975..78b1c83b 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -24,6 +24,7 @@ def find_root(): if "avoid-pep8-E402-import-not-top-of-file": # Use the util functions from cloudinit sys.path.insert(0, find_root()) + from cloudinit import subp from cloudinit import util from cloudinit import temp_utils from cloudinit import templater @@ -53,7 +54,7 @@ def run_helper(helper, args=None, strip=True): if args is None: args = [] cmd = [util.abs_join(find_root(), 'tools', helper)] + args - (stdout, _stderr) = util.subp(cmd) + (stdout, _stderr) = subp.subp(cmd) if strip: stdout = stdout.strip() return stdout @@ -67,7 +68,7 @@ def write_debian_folder(root, templ_data, cloud_util_deps): # Just copy debian/ dir and then update files pdeb_d = util.abs_join(find_root(), 'packages', 'debian') - util.subp(['cp', '-a', pdeb_d, deb_dir]) + subp.subp(['cp', '-a', pdeb_d, deb_dir]) # Fill in the change log template templater.render_to_file(util.abs_join(find_root(), @@ -190,7 +191,7 @@ def main(): print("Extracting temporary tarball %r" % (tarball)) cmd = ['tar', '-xvzf', tarball_fp, '-C', tdir] - util.subp(cmd, capture=capture) + subp.subp(cmd, capture=capture) xdir = util.abs_join(tdir, "cloud-init-%s" % ver_data['version_long']) templ_data.update(ver_data) @@ -203,7 +204,7 @@ def main(): cmd = ['debuild', '--preserve-envvar', 'INIT_SYSTEM'] if args.debuild_args: cmd.extend(args.debuild_args) - util.subp(cmd, capture=capture) + subp.subp(cmd, capture=capture) link_fn = os.path.join(os.getcwd(), 'cloud-init_all.deb') link_dsc = os.path.join(os.getcwd(), 'cloud-init.dsc') diff --git a/packages/brpm b/packages/brpm index 1be8804c..56477d2e 100755 --- a/packages/brpm +++ b/packages/brpm @@ -24,6 +24,7 @@ def find_root(): if "avoid-pep8-E402-import-not-top-of-file": # Use the util functions from cloudinit sys.path.insert(0, find_root()) + from cloudinit import subp from cloudinit import templater from cloudinit import util @@ -36,7 +37,7 @@ def run_helper(helper, args=None, strip=True): if args is None: args = [] cmd = [util.abs_join(find_root(), 'tools', helper)] + args - (stdout, _stderr) = util.subp(cmd) + (stdout, _stderr) = subp.subp(cmd) if strip: stdout = stdout.strip() return stdout @@ -174,7 +175,7 @@ def main(): else: cmd = ['rpmbuild', '-ba', spec_fn] - util.subp(cmd, capture=capture) + subp.subp(cmd, capture=capture) # Copy the items built to our local dir globs = [] diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py index f04d0cd4..6f74436d 100644 --- a/tests/cloud_tests/bddeb.py +++ b/tests/cloud_tests/bddeb.py @@ -6,7 +6,7 @@ from functools import partial import os import tempfile -from cloudinit import util as c_util +from cloudinit import subp from tests.cloud_tests import (config, LOG) from tests.cloud_tests import platforms from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single) @@ -42,8 +42,8 @@ def build_deb(args, instance): 'GIT_WORK_TREE': extract_dir} LOG.debug('creating tarball of cloud-init at: %s', local_tarball) - c_util.subp(['tar', 'cf', local_tarball, '--owner', 'root', - '--group', 'root', '-C', args.cloud_init, '.']) + subp.subp(['tar', 'cf', local_tarball, '--owner', 'root', + '--group', 'root', '-C', args.cloud_init, '.']) LOG.debug('copying to remote system at: %s', remote_tarball) instance.push_file(local_tarball, remote_tarball) diff --git a/tests/cloud_tests/platforms/lxd/image.py b/tests/cloud_tests/platforms/lxd/image.py index b5de1f52..8934fb74 100644 --- a/tests/cloud_tests/platforms/lxd/image.py +++ b/tests/cloud_tests/platforms/lxd/image.py @@ -8,6 +8,7 @@ import tempfile from ..images import Image from .snapshot import LXDSnapshot +from cloudinit import subp from cloudinit import util as c_util from tests.cloud_tests import util @@ -81,8 +82,8 @@ class LXDImage(Image): @return_value: tuple of path to metadata tarball and rootfs tarball """ # pylxd's image export feature doesn't do split exports, so use cmdline - c_util.subp(['lxc', 'image', 'export', self.pylxd_image.fingerprint, - output_dir], capture=True) + subp.subp(['lxc', 'image', 'export', self.pylxd_image.fingerprint, + output_dir], capture=True) tarballs = [p for p in os.listdir(output_dir) if p.endswith('tar.xz')] metadata = os.path.join( output_dir, next(p for p in tarballs if p.startswith('meta-'))) @@ -101,8 +102,8 @@ class LXDImage(Image): """ alias = util.gen_instance_name( image_desc=str(self), use_desc='update-metadata') - c_util.subp(['lxc', 'image', 'import', metadata, rootfs, - '--alias', alias], capture=True) + subp.subp(['lxc', 'image', 'import', metadata, rootfs, + '--alias', alias], capture=True) self.pylxd_image = self.platform.query_image_by_alias(alias) return self.pylxd_image.fingerprint diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py index 2b804a62..b27b9848 100644 --- a/tests/cloud_tests/platforms/lxd/instance.py +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -7,7 +7,8 @@ import shutil import time from tempfile import mkdtemp -from cloudinit.util import load_yaml, subp, ProcessExecutionError, which +from cloudinit.subp import subp, ProcessExecutionError, which +from cloudinit.util import load_yaml from tests.cloud_tests import LOG from tests.cloud_tests.util import PlatformError diff --git a/tests/cloud_tests/platforms/nocloudkvm/image.py b/tests/cloud_tests/platforms/nocloudkvm/image.py index bc2b6e75..ff5b6ad7 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/image.py +++ b/tests/cloud_tests/platforms/nocloudkvm/image.py @@ -2,7 +2,7 @@ """NoCloud KVM Image Base Class.""" -from cloudinit import util as c_util +from cloudinit import subp import os import shutil @@ -30,8 +30,8 @@ class NoCloudKVMImage(Image): self._img_path = os.path.join(self._workd, os.path.basename(self._orig_img_path)) - c_util.subp(['qemu-img', 'create', '-f', 'qcow2', - '-b', orig_img_path, self._img_path]) + subp.subp(['qemu-img', 'create', '-f', 'qcow2', + '-b', orig_img_path, self._img_path]) super(NoCloudKVMImage, self).__init__(platform, config) @@ -50,10 +50,10 @@ class NoCloudKVMImage(Image): '--system-resolvconf', self._img_path, '--', 'chroot', '_MOUNTPOINT_'] try: - out, err = c_util.subp(mic_chroot + env_args + list(command), - data=stdin, decode=False) + out, err = subp.subp(mic_chroot + env_args + list(command), + data=stdin, decode=False) return (out, err, 0) - except c_util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: return (e.stdout, e.stderr, e.exit_code) def snapshot(self): diff --git a/tests/cloud_tests/platforms/nocloudkvm/instance.py b/tests/cloud_tests/platforms/nocloudkvm/instance.py index 96185b75..5140a11c 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/instance.py +++ b/tests/cloud_tests/platforms/nocloudkvm/instance.py @@ -11,7 +11,7 @@ import uuid from ..instances import Instance from cloudinit.atomic_helper import write_json -from cloudinit import util as c_util +from cloudinit import subp from tests.cloud_tests import LOG, util # This domain contains reverse lookups for hostnames that are used. @@ -110,8 +110,8 @@ class NoCloudKVMInstance(Instance): """Clean up instance.""" if self.pid: try: - c_util.subp(['kill', '-9', self.pid]) - except c_util.ProcessExecutionError: + subp.subp(['kill', '-9', self.pid]) + except subp.ProcessExecutionError: pass if self.pid_file: @@ -143,8 +143,8 @@ class NoCloudKVMInstance(Instance): # meta-data can be yaml, but more easily pretty printed with json write_json(meta_data_file, self.meta_data) - c_util.subp(['cloud-localds', seed_file, user_data_file, - meta_data_file]) + subp.subp(['cloud-localds', seed_file, user_data_file, + meta_data_file]) return seed_file diff --git a/tests/cloud_tests/platforms/nocloudkvm/platform.py b/tests/cloud_tests/platforms/nocloudkvm/platform.py index 2d1480f5..53c8ebf2 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/platform.py +++ b/tests/cloud_tests/platforms/nocloudkvm/platform.py @@ -12,6 +12,7 @@ from simplestreams import util as s_util from ..platforms import Platform from .image import NoCloudKVMImage from .instance import NoCloudKVMInstance +from cloudinit import subp from cloudinit import util as c_util from tests.cloud_tests import util @@ -84,8 +85,8 @@ class NoCloudKVMPlatform(Platform): """ name = util.gen_instance_name(image_desc=image_desc, use_desc=use_desc) img_path = os.path.join(self.config['data_dir'], name + '.qcow2') - c_util.subp(['qemu-img', 'create', '-f', 'qcow2', - '-b', src_img_path, img_path]) + subp.subp(['qemu-img', 'create', '-f', 'qcow2', + '-b', src_img_path, img_path]) return NoCloudKVMInstance(self, name, img_path, properties, config, features, user_data, meta_data) diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py index bebdf1c6..58f65e52 100644 --- a/tests/cloud_tests/platforms/platforms.py +++ b/tests/cloud_tests/platforms/platforms.py @@ -7,6 +7,7 @@ import shutil from simplestreams import filters, mirrors from simplestreams import util as s_util +from cloudinit import subp from cloudinit import util as c_util from tests.cloud_tests import util @@ -48,10 +49,10 @@ class Platform(object): if os.path.exists(filename): c_util.del_file(filename) - c_util.subp(['ssh-keygen', '-m', 'PEM', '-t', 'rsa', '-b', '4096', - '-f', filename, '-P', '', - '-C', 'ubuntu@cloud_test'], - capture=True) + subp.subp(['ssh-keygen', '-m', 'PEM', '-t', 'rsa', '-b', '4096', + '-f', filename, '-P', '', + '-C', 'ubuntu@cloud_test'], + capture=True) @staticmethod def _query_streams(img_conf, img_filter): diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py index e65771b1..7dcccbdd 100644 --- a/tests/cloud_tests/util.py +++ b/tests/cloud_tests/util.py @@ -17,6 +17,7 @@ import time import yaml from contextlib import contextmanager +from cloudinit import subp from cloudinit import util as c_util from tests.cloud_tests import LOG @@ -232,8 +233,8 @@ def flat_tar(output, basedir, owner='root', group='root'): @param group: group archive files belong to @return_value: none """ - c_util.subp(['tar', 'cf', output, '--owner', owner, '--group', group, - '-C', basedir] + rel_files(basedir), capture=True) + subp.subp(['tar', 'cf', output, '--owner', owner, '--group', group, + '-C', basedir] + rel_files(basedir), capture=True) def parse_conf_list(entries, valid=None, boolean=False): @@ -465,7 +466,7 @@ class TargetBase(object): return path -class InTargetExecuteError(c_util.ProcessExecutionError): +class InTargetExecuteError(subp.ProcessExecutionError): """Error type for in target commands that fail.""" default_desc = 'Unexpected error while running command.' diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 9045e743..c5675249 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -15,6 +15,7 @@ from cloudinit.tests.helpers import ( from cloudinit import handlers from cloudinit import helpers +from cloudinit import subp from cloudinit import util from cloudinit.handlers.cloud_config import CloudConfigPartHandler @@ -66,7 +67,7 @@ class TestUpstartJobPartHandler(FilesystemMockingTestCase): util.ensure_dir("/etc/upstart") with mock.patch(self.mpath + 'SUITABLE_UPSTART', return_value=True): - with mock.patch.object(util, 'subp') as m_subp: + with mock.patch.object(subp, 'subp') as m_subp: h = UpstartJobPartHandler(paths) h.handle_part('', handlers.CONTENT_START, None, None, None) diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index 3119bfac..fc59d1d5 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -15,6 +15,7 @@ import shutil import tempfile from cloudinit import helpers +from cloudinit import subp from cloudinit import util from cloudinit.tests.helpers import CiTestCase, mock @@ -286,7 +287,7 @@ class TestUserDataRhevm(CiTestCase): def test_modprobe_fails(self): '''Test user_data_rhevm() where modprobe fails.''' - self.m_modprobe_floppy.side_effect = util.ProcessExecutionError( + self.m_modprobe_floppy.side_effect = subp.ProcessExecutionError( "Failed modprobe") dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual(False, dsrc.user_data_rhevm()) @@ -294,7 +295,7 @@ class TestUserDataRhevm(CiTestCase): def test_no_modprobe_cmd(self): '''Test user_data_rhevm() with no modprobe command.''' - self.m_modprobe_floppy.side_effect = util.ProcessExecutionError( + self.m_modprobe_floppy.side_effect = subp.ProcessExecutionError( "No such file or dir") dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual(False, dsrc.user_data_rhevm()) @@ -302,7 +303,7 @@ class TestUserDataRhevm(CiTestCase): def test_udevadm_fails(self): '''Test user_data_rhevm() where udevadm fails.''' - self.m_udevadm_settle.side_effect = util.ProcessExecutionError( + self.m_udevadm_settle.side_effect = subp.ProcessExecutionError( "Failed settle.") dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual(False, dsrc.user_data_rhevm()) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index b9f4d8fd..05552a1e 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -491,7 +491,7 @@ scbus-1 on xpt0 bus 0 (dsaz, 'get_hostname', mock.MagicMock()), (dsaz, 'set_hostname', mock.MagicMock()), (dsaz, 'get_metadata_from_fabric', self.get_metadata_from_fabric), - (dsaz.util, 'which', lambda x: True), + (dsaz.subp, 'which', lambda x: True), (dsaz.util, 'read_dmi_data', mock.MagicMock( side_effect=_dmi_mocks)), (dsaz.util, 'wait_for_files', mock.MagicMock( @@ -1267,20 +1267,20 @@ scbus-1 on xpt0 bus 0 expected_config['config'].append(blacklist_config) self.assertEqual(netconfig, expected_config) - @mock.patch(MOCKPATH + 'util.subp') - def test_get_hostname_with_no_args(self, subp): + @mock.patch(MOCKPATH + 'subp.subp') + def test_get_hostname_with_no_args(self, m_subp): dsaz.get_hostname() - subp.assert_called_once_with(("hostname",), capture=True) + m_subp.assert_called_once_with(("hostname",), capture=True) - @mock.patch(MOCKPATH + 'util.subp') - def test_get_hostname_with_string_arg(self, subp): + @mock.patch(MOCKPATH + 'subp.subp') + def test_get_hostname_with_string_arg(self, m_subp): dsaz.get_hostname(hostname_command="hostname") - subp.assert_called_once_with(("hostname",), capture=True) + m_subp.assert_called_once_with(("hostname",), capture=True) - @mock.patch(MOCKPATH + 'util.subp') - def test_get_hostname_with_iterable_arg(self, subp): + @mock.patch(MOCKPATH + 'subp.subp') + def test_get_hostname_with_iterable_arg(self, m_subp): dsaz.get_hostname(hostname_command=("hostname",)) - subp.assert_called_once_with(("hostname",), capture=True) + m_subp.assert_called_once_with(("hostname",), capture=True) class TestAzureBounce(CiTestCase): @@ -1302,7 +1302,7 @@ class TestAzureBounce(CiTestCase): mock.patch.object(dsaz, 'get_metadata_from_imds', mock.MagicMock(return_value={}))) self.patches.enter_context( - mock.patch.object(dsaz.util, 'which', lambda x: True)) + mock.patch.object(dsaz.subp, 'which', lambda x: True)) self.patches.enter_context(mock.patch.object( dsaz, '_get_random_seed', return_value='wild')) @@ -1331,7 +1331,7 @@ class TestAzureBounce(CiTestCase): self.set_hostname = self.patches.enter_context( mock.patch.object(dsaz, 'set_hostname')) self.subp = self.patches.enter_context( - mock.patch(MOCKPATH + 'util.subp')) + mock.patch(MOCKPATH + 'subp.subp')) self.find_fallback_nic = self.patches.enter_context( mock.patch('cloudinit.net.find_fallback_nic', return_value='eth9')) @@ -1414,7 +1414,7 @@ class TestAzureBounce(CiTestCase): cfg = {'hostname_bounce': {'policy': 'force'}} dsrc = self._get_ds(self.get_ovf_env_with_dscfg(host_name, cfg), agent_command=['not', '__builtin__']) - patch_path = MOCKPATH + 'util.which' + patch_path = MOCKPATH + 'subp.which' with mock.patch(patch_path) as m_which: m_which.return_value = None ret = self._get_and_setup(dsrc) @@ -1988,7 +1988,7 @@ class TestPreprovisioningPollIMDS(CiTestCase): self.assertEqual(report_ready_func.call_count, 0) -@mock.patch(MOCKPATH + 'util.subp') +@mock.patch(MOCKPATH + 'subp.subp') @mock.patch(MOCKPATH + 'util.write_file') @mock.patch(MOCKPATH + 'util.is_FreeBSD') @mock.patch('cloudinit.sources.helpers.netlink.' @@ -2159,7 +2159,7 @@ class TestWBIsPlatformViable(CiTestCase): {'os.path.exists': False, # Non-matching Azure chassis-asset-tag 'util.read_dmi_data': dsaz.AZURE_CHASSIS_ASSET_TAG + 'X', - 'util.which': None}, + 'subp.which': None}, dsaz._is_platform_viable, 'doesnotmatter')) self.assertIn( "DEBUG: Non-Azure DMI asset tag '{0}' discovered.\n".format( diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 6344b0bb..71ef57f0 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -287,7 +287,7 @@ class TestOpenSSLManager(CiTestCase): self.addCleanup(patches.close) self.subp = patches.enter_context( - mock.patch.object(azure_helper.util, 'subp')) + mock.patch.object(azure_helper.subp, 'subp')) try: self.open = patches.enter_context( mock.patch('__builtin__.open')) diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index 83c2f753..e68168f2 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -41,7 +41,7 @@ class TestCloudStackPasswordFetching(CiTestCase): def _set_password_server_response(self, response_string): subp = mock.MagicMock(return_value=(response_string, '')) self.patches.enter_context( - mock.patch('cloudinit.sources.DataSourceCloudStack.util.subp', + mock.patch('cloudinit.sources.DataSourceCloudStack.subp.subp', subp)) return subp diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 486a2345..3ef7a4b2 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -10,6 +10,7 @@ import os from collections import OrderedDict from textwrap import dedent +from cloudinit import subp from cloudinit import util from cloudinit.tests.helpers import CiTestCase, mock, wrap_and_call from cloudinit.helpers import Paths @@ -401,8 +402,8 @@ class TestTransportIso9660(CiTestCase): self.assertTrue(dsovf.maybe_cdrom_device('xvdza1')) -@mock.patch(MPATH + "util.which") -@mock.patch(MPATH + "util.subp") +@mock.patch(MPATH + "subp.which") +@mock.patch(MPATH + "subp.subp") class TestTransportVmwareGuestinfo(CiTestCase): """Test the com.vmware.guestInfo transport implemented in transport_vmware_guestinfo.""" @@ -420,7 +421,7 @@ class TestTransportVmwareGuestinfo(CiTestCase): def test_notfound_on_exit_code_1(self, m_subp, m_which): """If vmware-rpctool exits 1, then must return not found.""" m_which.return_value = self.rpctool_path - m_subp.side_effect = util.ProcessExecutionError( + m_subp.side_effect = subp.ProcessExecutionError( stdout="", stderr="No value found", exit_code=1, cmd=["unused"]) self.assertEqual(NOT_FOUND, dsovf.transport_vmware_guestinfo()) self.assertEqual(1, m_subp.call_count) @@ -442,7 +443,7 @@ class TestTransportVmwareGuestinfo(CiTestCase): def test_notfound_and_warns_on_unexpected_exit_code(self, m_subp, m_which): """If vmware-rpctool exits non zero or 1, warnings should be logged.""" m_which.return_value = self.rpctool_path - m_subp.side_effect = util.ProcessExecutionError( + m_subp.side_effect = subp.ProcessExecutionError( stdout=None, stderr="No value found", exit_code=2, cmd=["unused"]) self.assertEqual(NOT_FOUND, dsovf.transport_vmware_guestinfo()) self.assertEqual(1, m_subp.call_count) diff --git a/tests/unittests/test_datasource/test_rbx.py b/tests/unittests/test_datasource/test_rbx.py index 553af62e..d017510e 100644 --- a/tests/unittests/test_datasource/test_rbx.py +++ b/tests/unittests/test_datasource/test_rbx.py @@ -4,7 +4,7 @@ from cloudinit import helpers from cloudinit import distros from cloudinit.sources import DataSourceRbxCloud as ds from cloudinit.tests.helpers import mock, CiTestCase, populate_dir -from cloudinit import util +from cloudinit import subp DS_PATH = "cloudinit.sources.DataSourceRbxCloud" @@ -157,7 +157,7 @@ class TestRbxDataSource(CiTestCase): expected ) - @mock.patch(DS_PATH + '.util.subp') + @mock.patch(DS_PATH + '.subp.subp') def test_gratuitous_arp_run_standard_arping(self, m_subp): """Test handle run arping & parameters.""" items = [ @@ -183,7 +183,7 @@ class TestRbxDataSource(CiTestCase): ], m_subp.call_args_list ) - @mock.patch(DS_PATH + '.util.subp') + @mock.patch(DS_PATH + '.subp.subp') def test_handle_rhel_like_arping(self, m_subp): """Test handle on RHEL-like distros.""" items = [ @@ -201,8 +201,8 @@ class TestRbxDataSource(CiTestCase): ) @mock.patch( - DS_PATH + '.util.subp', - side_effect=util.ProcessExecutionError() + DS_PATH + '.subp.subp', + side_effect=subp.ProcessExecutionError() ) def test_continue_on_arping_error(self, m_subp): """Continue when command error""" diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index f1ab1d7a..5847a384 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -32,8 +32,8 @@ from cloudinit.sources.DataSourceSmartOS import ( from cloudinit.event import EventType from cloudinit import helpers as c_helpers -from cloudinit.util import ( - b64e, subp, ProcessExecutionError, which, write_file) +from cloudinit.util import (b64e, write_file) +from cloudinit.subp import (subp, ProcessExecutionError, which) from cloudinit.tests.helpers import ( CiTestCase, mock, FilesystemMockingTestCase, skipIf) @@ -667,7 +667,7 @@ class TestIdentifyFile(CiTestCase): with self.allow_subp(["file"]): self.assertEqual("text/plain", identify_file(fname)) - @mock.patch(DSMOS + ".util.subp") + @mock.patch(DSMOS + ".subp.subp") def test_returns_none_on_error(self, m_subp): """On 'file' execution error, None should be returned.""" m_subp.side_effect = ProcessExecutionError("FILE_FAILED", exit_code=99) diff --git a/tests/unittests/test_distros/test_create_users.py b/tests/unittests/test_distros/test_create_users.py index ef11784d..94ab052d 100644 --- a/tests/unittests/test_distros/test_create_users.py +++ b/tests/unittests/test_distros/test_create_users.py @@ -46,7 +46,7 @@ class MyBaseDistro(distros.Distro): @mock.patch("cloudinit.distros.util.system_is_snappy", return_value=False) -@mock.patch("cloudinit.distros.util.subp") +@mock.patch("cloudinit.distros.subp.subp") class TestCreateUser(CiTestCase): with_logs = True @@ -240,7 +240,7 @@ class TestCreateUser(CiTestCase): [mock.call(set(['auth1']), user), # not disabled mock.call(set(['key1']), 'foouser', options=disable_prefix)]) - @mock.patch("cloudinit.distros.util.which") + @mock.patch("cloudinit.distros.subp.which") def test_lock_with_usermod_if_no_passwd(self, m_which, m_subp, m_is_snappy): """Lock uses usermod --lock if no 'passwd' cmd available.""" @@ -250,7 +250,7 @@ class TestCreateUser(CiTestCase): [mock.call(['usermod', '--lock', 'bob'])], m_subp.call_args_list) - @mock.patch("cloudinit.distros.util.which") + @mock.patch("cloudinit.distros.subp.which") def test_lock_with_passwd_if_available(self, m_which, m_subp, m_is_snappy): """Lock with only passwd will use passwd.""" @@ -260,7 +260,7 @@ class TestCreateUser(CiTestCase): [mock.call(['passwd', '-l', 'bob'])], m_subp.call_args_list) - @mock.patch("cloudinit.distros.util.which") + @mock.patch("cloudinit.distros.subp.which") def test_lock_raises_runtime_if_no_commands(self, m_which, m_subp, m_is_snappy): """Lock with no commands available raises RuntimeError.""" diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py index da16a797..7ff8240b 100644 --- a/tests/unittests/test_distros/test_debian.py +++ b/tests/unittests/test_distros/test_debian.py @@ -5,7 +5,7 @@ from cloudinit import util from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock) -@mock.patch("cloudinit.distros.debian.util.subp") +@mock.patch("cloudinit.distros.debian.subp.subp") class TestDebianApplyLocale(FilesystemMockingTestCase): def setUp(self): diff --git a/tests/unittests/test_distros/test_freebsd.py b/tests/unittests/test_distros/test_freebsd.py index 8af253a2..be565b04 100644 --- a/tests/unittests/test_distros/test_freebsd.py +++ b/tests/unittests/test_distros/test_freebsd.py @@ -8,7 +8,7 @@ import os class TestDeviceLookUp(CiTestCase): - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_find_freebsd_part_label(self, mock_subp): glabel_out = ''' gptid/fa52d426-c337-11e6-8911-00155d4c5e47 N/A da0p1 @@ -19,7 +19,7 @@ gptid/fa52d426-c337-11e6-8911-00155d4c5e47 N/A da0p1 res = find_freebsd_part("/dev/label/rootfs") self.assertEqual("da0p2", res) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_find_freebsd_part_gpt(self, mock_subp): glabel_out = ''' gpt/bootfs N/A vtbd0p1 diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index 02b334e3..6b6c1566 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -245,7 +245,7 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): for d_name in ("ubuntu", "rhel"): cls = distros.fetch(d_name) d = cls(d_name, {}, None) - with mock.patch("cloudinit.util.subp") as m_subp: + with mock.patch("cloudinit.subp.subp") as m_subp: d.expire_passwd("myuser") m_subp.assert_called_once_with(["passwd", "--expire", "myuser"]) @@ -253,7 +253,7 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): """Test FreeBSD.expire_passwd uses the pw command.""" cls = distros.fetch("freebsd") d = cls("freebsd", {}, None) - with mock.patch("cloudinit.util.subp") as m_subp: + with mock.patch("cloudinit.subp.subp") as m_subp: d.expire_passwd("myuser") m_subp.assert_called_once_with( ["pw", "usermod", "myuser", "-p", "01-Jan-1970"]) diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 910207ca..8d7b09c8 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -12,6 +12,7 @@ from cloudinit import helpers from cloudinit import settings from cloudinit.tests.helpers import ( FilesystemMockingTestCase, dir2dict) +from cloudinit import subp from cloudinit import util @@ -688,6 +689,6 @@ class TestNetCfgDistroArch(TestNetCfgDistroBase): def get_mode(path, target=None): - return os.stat(util.target_path(target, path)).st_mode & 0o777 + return os.stat(subp.target_path(target, path)).st_mode & 0o777 # vi: ts=4 expandtab diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index a6faf0ef..fa48410a 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -307,7 +307,7 @@ class TestUGNormalize(TestCase): self.assertEqual({'default': False}, users['joe']) self.assertEqual({'default': False}, users['bob']) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_create_snap_user(self, mock_subp): mock_subp.side_effect = [('{"username": "joe", "ssh-key-count": 1}\n', '')] @@ -326,7 +326,7 @@ class TestUGNormalize(TestCase): mock_subp.assert_called_with(snapcmd, capture=True, logstring=snapcmd) self.assertEqual(username, 'joe') - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_create_snap_user_known(self, mock_subp): mock_subp.side_effect = [('{"username": "joe", "ssh-key-count": 1}\n', '')] @@ -348,7 +348,7 @@ class TestUGNormalize(TestCase): @mock.patch('cloudinit.util.system_is_snappy') @mock.patch('cloudinit.util.is_group') - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_add_user_on_snappy_system(self, mock_subp, mock_isgrp, mock_snappy): mock_isgrp.return_value = False diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 65a96090..fbb88e8a 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -6,6 +6,7 @@ import os from uuid import uuid4 from cloudinit import safeyaml +from cloudinit import subp from cloudinit import util from cloudinit.tests.helpers import ( CiTestCase, dir2dict, populate_dir, populate_dir_with_ts) @@ -160,8 +161,8 @@ class DsIdentifyBase(CiTestCase): rc = 0 try: - out, err = util.subp(['sh', '-c', '. %s' % wrap], capture=True) - except util.ProcessExecutionError as e: + out, err = subp.subp(['sh', '-c', '. %s' % wrap], capture=True) + except subp.ProcessExecutionError as e: rc = e.exit_code out = e.stdout err = e.stderr diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py index 69009a44..e5382544 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py @@ -13,6 +13,7 @@ from cloudinit import cloud from cloudinit import distros from cloudinit import helpers from cloudinit import templater +from cloudinit import subp from cloudinit import util from cloudinit.config import cc_apt_configure @@ -66,7 +67,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): """ def setUp(self): super(TestAptSourceConfigSourceList, self).setUp() - self.subp = util.subp + self.subp = subp.subp self.new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.new_root) @@ -176,7 +177,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # the second mock restores the original subp with mock.patch.object(util, 'write_file') as mockwrite: - with mock.patch.object(util, 'subp', self.subp): + with mock.patch.object(subp, 'subp', self.subp): with mock.patch.object(Distro, 'get_primary_arch', return_value='amd64'): cc_apt_configure.handle("notimportant", cfg, mycloud, diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py index 0aa3d51a..b96fd4d4 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py @@ -13,6 +13,7 @@ from unittest.mock import call from cloudinit import cloud from cloudinit import distros from cloudinit import helpers +from cloudinit import subp from cloudinit import util from cloudinit.config import cc_apt_configure @@ -94,7 +95,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): """TestAptSourceConfigSourceList - Class to test sources list rendering""" def setUp(self): super(TestAptSourceConfigSourceList, self).setUp() - self.subp = util.subp + self.subp = subp.subp self.new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.new_root) @@ -222,7 +223,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # the second mock restores the original subp with mock.patch.object(util, 'write_file') as mockwrite: - with mock.patch.object(util, 'subp', self.subp): + with mock.patch.object(subp, 'subp', self.subp): with mock.patch.object(Distro, 'get_primary_arch', return_value='amd64'): cc_apt_configure.handle("notimportant", cfg, mycloud, diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py index 866752ef..f2349157 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py @@ -14,6 +14,7 @@ from unittest.mock import call from cloudinit.config import cc_apt_configure from cloudinit import gpg +from cloudinit import subp from cloudinit import util from cloudinit.tests.helpers import TestCase @@ -271,7 +272,7 @@ class TestAptSourceConfig(TestCase): """ cfg = self.wrapv1conf(cfg) - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('fakekey 1234', '')) as mockobj: cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) @@ -356,7 +357,7 @@ class TestAptSourceConfig(TestCase): """ cfg = self.wrapv1conf([cfg]) - with mock.patch.object(util, 'subp') as mockobj: + with mock.patch.object(subp, 'subp') as mockobj: cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) mockobj.assert_called_with(['apt-key', 'add', '-'], @@ -398,7 +399,7 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile} cfg = self.wrapv1conf([cfg]) - with mock.patch.object(util, 'subp') as mockobj: + with mock.patch.object(subp, 'subp') as mockobj: cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) mockobj.assert_called_once_with(['apt-key', 'add', '-'], @@ -413,7 +414,7 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile} cfg = self.wrapv1conf([cfg]) - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('fakekey 1212', '')) as mockobj: cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) @@ -476,7 +477,7 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile} cfg = self.wrapv1conf([cfg]) - with mock.patch.object(util, 'subp') as mockobj: + with mock.patch.object(subp, 'subp') as mockobj: cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) mockobj.assert_called_once_with(['add-apt-repository', 'ppa:smoser/cloud-init-test'], @@ -495,7 +496,7 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile3} cfg = self.wrapv1conf([cfg1, cfg2, cfg3]) - with mock.patch.object(util, 'subp') as mockobj: + with mock.patch.object(subp, 'subp') as mockobj: cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test'], diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py index aefe26c4..220100e2 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -18,6 +18,7 @@ from cloudinit import cloud from cloudinit import distros from cloudinit import gpg from cloudinit import helpers +from cloudinit import subp from cloudinit import util from cloudinit.config import cc_apt_configure @@ -221,7 +222,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): """ params = self._get_default_params() - with mock.patch("cloudinit.util.subp", + with mock.patch("cloudinit.subp.subp", return_value=('fakekey 1234', '')) as mockobj: self._add_apt_sources(cfg, TARGET, template_params=params, aa_repo_match=self.matcher) @@ -296,7 +297,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): ' xenial main'), 'key': "fakekey 4321"}} - with mock.patch.object(util, 'subp') as mockobj: + with mock.patch.object(subp, 'subp') as mockobj: self._add_apt_sources(cfg, TARGET, template_params=params, aa_repo_match=self.matcher) @@ -318,7 +319,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): params = self._get_default_params() cfg = {self.aptlistfile: {'key': "fakekey 4242"}} - with mock.patch.object(util, 'subp') as mockobj: + with mock.patch.object(subp, 'subp') as mockobj: self._add_apt_sources(cfg, TARGET, template_params=params, aa_repo_match=self.matcher) @@ -333,7 +334,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): params = self._get_default_params() cfg = {self.aptlistfile: {'keyid': "03683F77"}} - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('fakekey 1212', '')) as mockobj: self._add_apt_sources(cfg, TARGET, template_params=params, aa_repo_match=self.matcher) @@ -416,7 +417,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): params = self._get_default_params() cfg = {self.aptlistfile: {'source': 'ppa:smoser/cloud-init-test'}} - with mock.patch("cloudinit.util.subp") as mockobj: + with mock.patch("cloudinit.subp.subp") as mockobj: self._add_apt_sources(cfg, TARGET, template_params=params, aa_repo_match=self.matcher) mockobj.assert_any_call(['add-apt-repository', @@ -432,7 +433,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): self.aptlistfile2: {'source': 'ppa:smoser/cloud-init-test2'}, self.aptlistfile3: {'source': 'ppa:smoser/cloud-init-test3'}} - with mock.patch("cloudinit.util.subp") as mockobj: + with mock.patch("cloudinit.subp.subp") as mockobj: self._add_apt_sources(cfg, TARGET, template_params=params, aa_repo_match=self.matcher) calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test'], @@ -996,7 +997,7 @@ deb http://ubuntu.com/ubuntu/ xenial-proposed main""") class TestDebconfSelections(TestCase): - @mock.patch("cloudinit.config.cc_apt_configure.util.subp") + @mock.patch("cloudinit.config.cc_apt_configure.subp.subp") def test_set_sel_appends_newline_if_absent(self, m_subp): """Automatically append a newline to debconf-set-selections config.""" selections = b'some/setting boolean true' @@ -1081,7 +1082,7 @@ class TestDebconfSelections(TestCase): self.assertTrue(m_get_inst.called) self.assertEqual(m_dpkg_r.call_count, 0) - @mock.patch("cloudinit.config.cc_apt_configure.util.subp") + @mock.patch("cloudinit.config.cc_apt_configure.subp.subp") def test_dpkg_reconfigure_does_reconfigure(self, m_subp): target = "/foo-target" @@ -1104,12 +1105,12 @@ class TestDebconfSelections(TestCase): 'cloud-init'] self.assertEqual(expected, found) - @mock.patch("cloudinit.config.cc_apt_configure.util.subp") + @mock.patch("cloudinit.config.cc_apt_configure.subp.subp") def test_dpkg_reconfigure_not_done_on_no_data(self, m_subp): cc_apt_configure.dpkg_reconfigure([]) m_subp.assert_not_called() - @mock.patch("cloudinit.config.cc_apt_configure.util.subp") + @mock.patch("cloudinit.config.cc_apt_configure.subp.subp") def test_dpkg_reconfigure_not_done_if_no_cleaners(self, m_subp): cc_apt_configure.dpkg_reconfigure(['pkgfoo', 'pkgbar']) m_subp.assert_not_called() diff --git a/tests/unittests/test_handler/test_handler_bootcmd.py b/tests/unittests/test_handler/test_handler_bootcmd.py index a76760fa..b53d60d4 100644 --- a/tests/unittests/test_handler/test_handler_bootcmd.py +++ b/tests/unittests/test_handler/test_handler_bootcmd.py @@ -2,7 +2,7 @@ from cloudinit.config.cc_bootcmd import handle, schema from cloudinit.sources import DataSourceNone -from cloudinit import (distros, helpers, cloud, util) +from cloudinit import (distros, helpers, cloud, subp, util) from cloudinit.tests.helpers import ( CiTestCase, mock, SchemaTestCaseMixin, skipUnlessJsonSchema) @@ -36,7 +36,7 @@ class TestBootcmd(CiTestCase): def setUp(self): super(TestBootcmd, self).setUp() - self.subp = util.subp + self.subp = subp.subp self.new_root = self.tmp_dir() def _get_cloud(self, distro): @@ -130,7 +130,7 @@ class TestBootcmd(CiTestCase): with mock.patch(self._etmpfile_path, FakeExtendedTempFile): with self.allow_subp(['/bin/sh']): - with self.assertRaises(util.ProcessExecutionError) as ctxt: + with self.assertRaises(subp.ProcessExecutionError) as ctxt: handle('does-not-matter', valid_config, cc, LOG, []) self.assertIn( 'Unexpected error while running command.\n' diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index 286ef771..db0cdf9b 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -3,6 +3,7 @@ from cloudinit import cloud from cloudinit.config import cc_ca_certs from cloudinit import helpers +from cloudinit import subp from cloudinit import util from cloudinit.tests.helpers import TestCase @@ -228,7 +229,7 @@ class TestAddCaCerts(TestCase): class TestUpdateCaCerts(unittest.TestCase): def test_commands(self): - with mock.patch.object(util, 'subp') as mockobj: + with mock.patch.object(subp, 'subp') as mockobj: cc_ca_certs.update_ca_certs() mockobj.assert_called_once_with( ["update-ca-certificates"], capture=False) @@ -250,7 +251,7 @@ class TestRemoveDefaultCaCerts(TestCase): mock.patch.object(util, 'delete_dir_contents')) mock_write = mocks.enter_context( mock.patch.object(util, 'write_file')) - mock_subp = mocks.enter_context(mock.patch.object(util, 'subp')) + mock_subp = mocks.enter_context(mock.patch.object(subp, 'subp')) cc_ca_certs.remove_default_ca_certs() diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 8c476418..7918c609 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -41,7 +41,7 @@ class TestInstallChefOmnibus(HttprettyTestCase): httpretty.GET, cc_chef.OMNIBUS_URL, body=response, status=200) ret = (None, None) # stdout, stderr but capture=False - with mock.patch("cloudinit.config.cc_chef.util.subp_blob_in_tempfile", + with mock.patch("cloudinit.config.cc_chef.subp_blob_in_tempfile", return_value=ret) as m_subp_blob: cc_chef.install_chef_from_omnibus() # admittedly whitebox, but assuming subp_blob_in_tempfile works @@ -52,7 +52,7 @@ class TestInstallChefOmnibus(HttprettyTestCase): m_subp_blob.call_args_list) @mock.patch('cloudinit.config.cc_chef.url_helper.readurl') - @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile') + @mock.patch('cloudinit.config.cc_chef.subp_blob_in_tempfile') def test_install_chef_from_omnibus_retries_url(self, m_subp_blob, m_rdurl): """install_chef_from_omnibus retries OMNIBUS_URL upon failure.""" @@ -81,7 +81,7 @@ class TestInstallChefOmnibus(HttprettyTestCase): m_subp_blob.call_args_list[0][1]) @mock.patch("cloudinit.config.cc_chef.OMNIBUS_URL", OMNIBUS_URL_HTTP) - @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile') + @mock.patch('cloudinit.config.cc_chef.subp_blob_in_tempfile') def test_install_chef_from_omnibus_has_omnibus_version(self, m_subp_blob): """install_chef_from_omnibus provides version arg to OMNIBUS_URL.""" chef_outfile = self.tmp_path('chef.out', self.new_root) diff --git a/tests/unittests/test_handler/test_handler_disk_setup.py b/tests/unittests/test_handler/test_handler_disk_setup.py index 0e51f17a..4f4a57fa 100644 --- a/tests/unittests/test_handler/test_handler_disk_setup.py +++ b/tests/unittests/test_handler/test_handler_disk_setup.py @@ -44,7 +44,7 @@ class TestGetMbrHddSize(TestCase): super(TestGetMbrHddSize, self).setUp() self.patches = ExitStack() self.subp = self.patches.enter_context( - mock.patch.object(cc_disk_setup.util, 'subp')) + mock.patch.object(cc_disk_setup.subp, 'subp')) def tearDown(self): super(TestGetMbrHddSize, self).tearDown() @@ -173,7 +173,7 @@ class TestUpdateFsSetupDevices(TestCase): @mock.patch('cloudinit.config.cc_disk_setup.find_device_node', return_value=('/dev/xdb1', False)) @mock.patch('cloudinit.config.cc_disk_setup.device_type', return_value=None) -@mock.patch('cloudinit.config.cc_disk_setup.util.subp', return_value=('', '')) +@mock.patch('cloudinit.config.cc_disk_setup.subp.subp', return_value=('', '')) class TestMkfsCommandHandling(CiTestCase): with_logs = True @@ -204,7 +204,7 @@ class TestMkfsCommandHandling(CiTestCase): subp.assert_called_once_with( 'mkfs -t ext4 -L with_cmd /dev/xdb1', shell=True) - @mock.patch('cloudinit.config.cc_disk_setup.util.which') + @mock.patch('cloudinit.config.cc_disk_setup.subp.which') def test_overwrite_and_extra_opts_without_cmd(self, m_which, subp, *args): """mkfs observes extra_opts and overwrite settings when cmd is not present.""" @@ -222,7 +222,7 @@ class TestMkfsCommandHandling(CiTestCase): '-L', 'without_cmd', '-F', 'are', 'added'], shell=False) - @mock.patch('cloudinit.config.cc_disk_setup.util.which') + @mock.patch('cloudinit.config.cc_disk_setup.subp.which') def test_mkswap(self, m_which, subp, *args): """mkfs observes extra_opts and overwrite settings when cmd is not present.""" diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index 501bcca5..7f039b79 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -2,7 +2,7 @@ from cloudinit import cloud from cloudinit.config import cc_growpart -from cloudinit import util +from cloudinit import subp from cloudinit.tests.helpers import TestCase @@ -95,7 +95,7 @@ class TestConfig(TestCase): @mock.patch.dict("os.environ", clear=True) def test_no_resizers_auto_is_fine(self): with mock.patch.object( - util, 'subp', + subp, 'subp', return_value=(HELP_GROWPART_NO_RESIZE, "")) as mockobj: config = {'growpart': {'mode': 'auto'}} @@ -109,7 +109,7 @@ class TestConfig(TestCase): @mock.patch.dict("os.environ", clear=True) def test_no_resizers_mode_growpart_is_exception(self): with mock.patch.object( - util, 'subp', + subp, 'subp', return_value=(HELP_GROWPART_NO_RESIZE, "")) as mockobj: config = {'growpart': {'mode': "growpart"}} self.assertRaises( @@ -122,7 +122,7 @@ class TestConfig(TestCase): @mock.patch.dict("os.environ", clear=True) def test_mode_auto_prefers_growpart(self): with mock.patch.object( - util, 'subp', + subp, 'subp', return_value=(HELP_GROWPART_RESIZE, "")) as mockobj: ret = cc_growpart.resizer_factory(mode="auto") self.assertIsInstance(ret, cc_growpart.ResizeGrowPart) @@ -133,7 +133,7 @@ class TestConfig(TestCase): @mock.patch.dict("os.environ", clear=True) def test_mode_auto_falls_back_to_gpart(self): with mock.patch.object( - util, 'subp', + subp, 'subp', return_value=("", HELP_GPART)) as mockobj: ret = cc_growpart.resizer_factory(mode="auto") self.assertIsInstance(ret, cc_growpart.ResizeGpart) diff --git a/tests/unittests/test_handler/test_handler_landscape.py b/tests/unittests/test_handler/test_handler_landscape.py index db92a7e2..7d165687 100644 --- a/tests/unittests/test_handler/test_handler_landscape.py +++ b/tests/unittests/test_handler/test_handler_landscape.py @@ -49,8 +49,8 @@ class TestLandscape(FilesystemMockingTestCase): "'landscape' key existed in config, but not a dict", str(context_manager.exception)) - @mock.patch('cloudinit.config.cc_landscape.util') - def test_handler_restarts_landscape_client(self, m_util): + @mock.patch('cloudinit.config.cc_landscape.subp') + def test_handler_restarts_landscape_client(self, m_subp): """handler restarts lansdscape-client after install.""" mycloud = self._get_cloud('ubuntu') cfg = {'landscape': {'client': {}}} @@ -60,7 +60,7 @@ class TestLandscape(FilesystemMockingTestCase): cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None) self.assertEqual( [mock.call(['service', 'landscape-client', 'restart'])], - m_util.subp.call_args_list) + m_subp.subp.call_args_list) def test_handler_installs_client_and_creates_config_file(self): """Write landscape client.conf and install landscape-client.""" diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index 407aa6c4..47e7d804 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -84,7 +84,7 @@ class TestLocale(t_help.FilesystemMockingTestCase): util.write_file(locale_conf, 'LANG="en_US.UTF-8"\n') cfg = {'locale': 'C.UTF-8'} cc = self._get_cloud('ubuntu') - with mock.patch('cloudinit.distros.debian.util.subp') as m_subp: + with mock.patch('cloudinit.distros.debian.subp.subp') as m_subp: with mock.patch('cloudinit.distros.debian.LOCALE_CONF_FN', locale_conf): cc_locale.handle('cc_locale', cfg, cc, LOG, []) diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 40b521e5..21011204 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -31,13 +31,13 @@ class TestLxd(t_help.CiTestCase): return cc @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") - @mock.patch("cloudinit.config.cc_lxd.util") - def test_lxd_init(self, mock_util, m_maybe_clean): + @mock.patch("cloudinit.config.cc_lxd.subp") + def test_lxd_init(self, mock_subp, m_maybe_clean): cc = self._get_cloud('ubuntu') - mock_util.which.return_value = True + mock_subp.which.return_value = True m_maybe_clean.return_value = None cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, []) - self.assertTrue(mock_util.which.called) + self.assertTrue(mock_subp.which.called) # no bridge config, so maybe_cleanup should not be called. self.assertFalse(m_maybe_clean.called) self.assertEqual( @@ -45,14 +45,14 @@ class TestLxd(t_help.CiTestCase): mock.call( ['lxd', 'init', '--auto', '--network-address=0.0.0.0', '--storage-backend=zfs', '--storage-pool=poolname'])], - mock_util.subp.call_args_list) + mock_subp.subp.call_args_list) @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") - @mock.patch("cloudinit.config.cc_lxd.util") - def test_lxd_install(self, mock_util, m_maybe_clean): + @mock.patch("cloudinit.config.cc_lxd.subp") + def test_lxd_install(self, mock_subp, m_maybe_clean): cc = self._get_cloud('ubuntu') cc.distro = mock.MagicMock() - mock_util.which.return_value = None + mock_subp.which.return_value = None cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, []) self.assertNotIn('WARN', self.logs.getvalue()) self.assertTrue(cc.distro.install_packages.called) @@ -62,23 +62,23 @@ class TestLxd(t_help.CiTestCase): self.assertEqual(sorted(install_pkg), ['lxd', 'zfsutils-linux']) @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") - @mock.patch("cloudinit.config.cc_lxd.util") - def test_no_init_does_nothing(self, mock_util, m_maybe_clean): + @mock.patch("cloudinit.config.cc_lxd.subp") + def test_no_init_does_nothing(self, mock_subp, m_maybe_clean): cc = self._get_cloud('ubuntu') cc.distro = mock.MagicMock() cc_lxd.handle('cc_lxd', {'lxd': {}}, cc, self.logger, []) self.assertFalse(cc.distro.install_packages.called) - self.assertFalse(mock_util.subp.called) + self.assertFalse(mock_subp.subp.called) self.assertFalse(m_maybe_clean.called) @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") - @mock.patch("cloudinit.config.cc_lxd.util") - def test_no_lxd_does_nothing(self, mock_util, m_maybe_clean): + @mock.patch("cloudinit.config.cc_lxd.subp") + def test_no_lxd_does_nothing(self, mock_subp, m_maybe_clean): cc = self._get_cloud('ubuntu') cc.distro = mock.MagicMock() cc_lxd.handle('cc_lxd', {'package_update': True}, cc, self.logger, []) self.assertFalse(cc.distro.install_packages.called) - self.assertFalse(mock_util.subp.called) + self.assertFalse(mock_subp.subp.called) self.assertFalse(m_maybe_clean.called) def test_lxd_debconf_new_full(self): diff --git a/tests/unittests/test_handler/test_handler_mcollective.py b/tests/unittests/test_handler/test_handler_mcollective.py index c013a538..6891e15f 100644 --- a/tests/unittests/test_handler/test_handler_mcollective.py +++ b/tests/unittests/test_handler/test_handler_mcollective.py @@ -136,8 +136,9 @@ class TestHandler(t_help.TestCase): cc = cloud.Cloud(ds, paths, {}, d, None) return cc + @t_help.mock.patch("cloudinit.config.cc_mcollective.subp") @t_help.mock.patch("cloudinit.config.cc_mcollective.util") - def test_mcollective_install(self, mock_util): + def test_mcollective_install(self, mock_util, mock_subp): cc = self._get_cloud('ubuntu') cc.distro = t_help.mock.MagicMock() mock_util.load_file.return_value = b"" @@ -147,8 +148,8 @@ class TestHandler(t_help.TestCase): install_pkg = cc.distro.install_packages.call_args_list[0][0][0] self.assertEqual(install_pkg, ('mcollective',)) - self.assertTrue(mock_util.subp.called) - self.assertEqual(mock_util.subp.call_args_list[0][0][0], + self.assertTrue(mock_subp.subp.called) + self.assertEqual(mock_subp.subp.call_args_list[0][0][0], ['service', 'mcollective', 'restart']) # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index 35e72bd1..80c53c83 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -155,8 +155,8 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase): 'mock_is_block_device', return_value=True) - self.add_patch('cloudinit.config.cc_mounts.util.subp', - 'm_util_subp') + self.add_patch('cloudinit.config.cc_mounts.subp.subp', + 'm_subp_subp') self.add_patch('cloudinit.config.cc_mounts.util.mounts', 'mock_util_mounts', @@ -268,7 +268,7 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase): fstab_new_content = fd.read() self.assertEqual(fstab_expected_content, fstab_new_content) cc_mounts.handle(None, cc, self.mock_cloud, self.mock_log, []) - self.m_util_subp.assert_has_calls([ + self.m_subp_subp.assert_has_calls([ mock.call(['mount', '-a']), mock.call(['systemctl', 'daemon-reload'])]) diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 463d892a..92a33ec1 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -83,50 +83,50 @@ class TestNtp(FilesystemMockingTestCase): ntpconfig['template_name'] = os.path.basename(confpath) return ntpconfig - @mock.patch("cloudinit.config.cc_ntp.util") - def test_ntp_install(self, mock_util): + @mock.patch("cloudinit.config.cc_ntp.subp") + def test_ntp_install(self, mock_subp): """ntp_install_client runs install_func when check_exe is absent.""" - mock_util.which.return_value = None # check_exe not found. + mock_subp.which.return_value = None # check_exe not found. install_func = mock.MagicMock() cc_ntp.install_ntp_client(install_func, packages=['ntpx'], check_exe='ntpdx') - mock_util.which.assert_called_with('ntpdx') + mock_subp.which.assert_called_with('ntpdx') install_func.assert_called_once_with(['ntpx']) - @mock.patch("cloudinit.config.cc_ntp.util") - def test_ntp_install_not_needed(self, mock_util): + @mock.patch("cloudinit.config.cc_ntp.subp") + def test_ntp_install_not_needed(self, mock_subp): """ntp_install_client doesn't install when check_exe is found.""" client = 'chrony' - mock_util.which.return_value = [client] # check_exe found. + mock_subp.which.return_value = [client] # check_exe found. install_func = mock.MagicMock() cc_ntp.install_ntp_client(install_func, packages=[client], check_exe=client) install_func.assert_not_called() - @mock.patch("cloudinit.config.cc_ntp.util") - def test_ntp_install_no_op_with_empty_pkg_list(self, mock_util): + @mock.patch("cloudinit.config.cc_ntp.subp") + def test_ntp_install_no_op_with_empty_pkg_list(self, mock_subp): """ntp_install_client runs install_func with empty list""" - mock_util.which.return_value = None # check_exe not found + mock_subp.which.return_value = None # check_exe not found install_func = mock.MagicMock() cc_ntp.install_ntp_client(install_func, packages=[], check_exe='timesyncd') install_func.assert_called_once_with([]) - @mock.patch("cloudinit.config.cc_ntp.util") - def test_reload_ntp_defaults(self, mock_util): + @mock.patch("cloudinit.config.cc_ntp.subp") + def test_reload_ntp_defaults(self, mock_subp): """Test service is restarted/reloaded (defaults)""" service = 'ntp_service_name' cmd = ['service', service, 'restart'] cc_ntp.reload_ntp(service) - mock_util.subp.assert_called_with(cmd, capture=True) + mock_subp.subp.assert_called_with(cmd, capture=True) - @mock.patch("cloudinit.config.cc_ntp.util") - def test_reload_ntp_systemd(self, mock_util): + @mock.patch("cloudinit.config.cc_ntp.subp") + def test_reload_ntp_systemd(self, mock_subp): """Test service is restarted/reloaded (systemd)""" service = 'ntp_service_name' cc_ntp.reload_ntp(service, systemd=True) cmd = ['systemctl', 'reload-or-restart', service] - mock_util.subp.assert_called_with(cmd, capture=True) + mock_subp.subp.assert_called_with(cmd, capture=True) def test_ntp_rename_ntp_conf(self): """When NTP_CONF exists, rename_ntp moves it.""" @@ -440,9 +440,10 @@ class TestNtp(FilesystemMockingTestCase): cc_ntp.handle('notimportant', cfg, mycloud, None, None) self.assertEqual(0, m_select.call_count) + @mock.patch("cloudinit.config.cc_ntp.subp") @mock.patch('cloudinit.config.cc_ntp.select_ntp_client') @mock.patch("cloudinit.distros.Distro.uses_systemd") - def test_ntp_the_whole_package(self, m_sysd, m_select): + def test_ntp_the_whole_package(self, m_sysd, m_select, m_subp): """Test enabled config renders template, and restarts service """ cfg = {'ntp': {'enabled': True}} for distro in cc_ntp.distros: @@ -458,12 +459,12 @@ class TestNtp(FilesystemMockingTestCase): # allow use of util.mergemanydict m_util.mergemanydict.side_effect = util.mergemanydict # default client is present - m_util.which.return_value = True + m_subp.which.return_value = True # use the config 'enabled' value m_util.is_false.return_value = util.is_false( cfg['ntp']['enabled']) cc_ntp.handle('notimportant', cfg, mycloud, None, None) - m_util.subp.assert_called_with( + m_subp.subp.assert_called_with( ['systemctl', 'reload-or-restart', service_name], capture=True) self.assertEqual( @@ -503,7 +504,7 @@ class TestNtp(FilesystemMockingTestCase): expected_client = mycloud.distro.preferred_ntp_clients[0] self.assertEqual('ntp', expected_client) - @mock.patch('cloudinit.config.cc_ntp.util.which') + @mock.patch('cloudinit.config.cc_ntp.subp.which') def test_snappy_system_picks_timesyncd(self, m_which): """Test snappy systems prefer installed clients""" @@ -528,7 +529,7 @@ class TestNtp(FilesystemMockingTestCase): self.assertEqual(sorted(expected_cfg), sorted(cfg)) self.assertEqual(sorted(expected_cfg), sorted(result)) - @mock.patch('cloudinit.config.cc_ntp.util.which') + @mock.patch('cloudinit.config.cc_ntp.subp.which') def test_ntp_distro_searches_all_preferred_clients(self, m_which): """Test select_ntp_client search all distro perferred clients """ # nothing is installed @@ -546,7 +547,7 @@ class TestNtp(FilesystemMockingTestCase): m_which.assert_has_calls(expected_calls) self.assertEqual(sorted(expected_cfg), sorted(cfg)) - @mock.patch('cloudinit.config.cc_ntp.util.which') + @mock.patch('cloudinit.config.cc_ntp.subp.which') def test_user_cfg_ntp_client_auto_uses_distro_clients(self, m_which): """Test user_cfg.ntp_client='auto' defaults to distro search""" # nothing is installed @@ -566,7 +567,7 @@ class TestNtp(FilesystemMockingTestCase): @mock.patch('cloudinit.config.cc_ntp.write_ntp_config_template') @mock.patch('cloudinit.cloud.Cloud.get_template_filename') - @mock.patch('cloudinit.config.cc_ntp.util.which') + @mock.patch('cloudinit.config.cc_ntp.subp.which') def test_ntp_custom_client_overrides_installed_clients(self, m_which, m_tmpfn, m_write): """Test user client is installed despite other clients present """ @@ -582,7 +583,7 @@ class TestNtp(FilesystemMockingTestCase): m_install.assert_called_with([client]) m_which.assert_called_with(client) - @mock.patch('cloudinit.config.cc_ntp.util.which') + @mock.patch('cloudinit.config.cc_ntp.subp.which') def test_ntp_system_config_overrides_distro_builtin_clients(self, m_which): """Test distro system_config overrides builtin preferred ntp clients""" system_client = 'chrony' @@ -597,7 +598,7 @@ class TestNtp(FilesystemMockingTestCase): self.assertEqual(sorted(expected_cfg), sorted(result)) m_which.assert_has_calls([]) - @mock.patch('cloudinit.config.cc_ntp.util.which') + @mock.patch('cloudinit.config.cc_ntp.subp.which') def test_ntp_user_config_overrides_system_cfg(self, m_which): """Test user-data overrides system_config ntp_client""" system_client = 'chrony' diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py index 2506d18a..62388ac6 100644 --- a/tests/unittests/test_handler/test_handler_puppet.py +++ b/tests/unittests/test_handler/test_handler_puppet.py @@ -12,11 +12,11 @@ import textwrap LOG = logging.getLogger(__name__) -@mock.patch('cloudinit.config.cc_puppet.util') +@mock.patch('cloudinit.config.cc_puppet.subp.subp') @mock.patch('cloudinit.config.cc_puppet.os') class TestAutostartPuppet(CiTestCase): - def test_wb_autostart_puppet_updates_puppet_default(self, m_os, m_util): + def test_wb_autostart_puppet_updates_puppet_default(self, m_os, m_subp): """Update /etc/default/puppet to autostart if it exists.""" def _fake_exists(path): @@ -27,9 +27,9 @@ class TestAutostartPuppet(CiTestCase): self.assertEqual( [mock.call(['sed', '-i', '-e', 's/^START=.*/START=yes/', '/etc/default/puppet'], capture=False)], - m_util.subp.call_args_list) + m_subp.call_args_list) - def test_wb_autostart_pupppet_enables_puppet_systemctl(self, m_os, m_util): + def test_wb_autostart_pupppet_enables_puppet_systemctl(self, m_os, m_subp): """If systemctl is present, enable puppet via systemctl.""" def _fake_exists(path): @@ -39,9 +39,9 @@ class TestAutostartPuppet(CiTestCase): cc_puppet._autostart_puppet(LOG) expected_calls = [mock.call( ['/bin/systemctl', 'enable', 'puppet.service'], capture=False)] - self.assertEqual(expected_calls, m_util.subp.call_args_list) + self.assertEqual(expected_calls, m_subp.call_args_list) - def test_wb_autostart_pupppet_enables_puppet_chkconfig(self, m_os, m_util): + def test_wb_autostart_pupppet_enables_puppet_chkconfig(self, m_os, m_subp): """If chkconfig is present, enable puppet via checkcfg.""" def _fake_exists(path): @@ -51,7 +51,7 @@ class TestAutostartPuppet(CiTestCase): cc_puppet._autostart_puppet(LOG) expected_calls = [mock.call( ['/sbin/chkconfig', 'puppet', 'on'], capture=False)] - self.assertEqual(expected_calls, m_util.subp.call_args_list) + self.assertEqual(expected_calls, m_subp.call_args_list) @mock.patch('cloudinit.config.cc_puppet._autostart_puppet') @@ -81,7 +81,7 @@ class TestPuppetHandle(CiTestCase): "no 'puppet' configuration found", self.logs.getvalue()) self.assertEqual(0, m_auto.call_count) - @mock.patch('cloudinit.config.cc_puppet.util.subp') + @mock.patch('cloudinit.config.cc_puppet.subp.subp') def test_handler_puppet_config_starts_puppet_service(self, m_subp, m_auto): """Cloud-config 'puppet' configuration starts puppet.""" mycloud = self._get_cloud('ubuntu') @@ -92,7 +92,7 @@ class TestPuppetHandle(CiTestCase): [mock.call(['service', 'puppet', 'start'], capture=False)], m_subp.call_args_list) - @mock.patch('cloudinit.config.cc_puppet.util.subp') + @mock.patch('cloudinit.config.cc_puppet.subp.subp') def test_handler_empty_puppet_config_installs_puppet(self, m_subp, m_auto): """Cloud-config empty 'puppet' configuration installs latest puppet.""" mycloud = self._get_cloud('ubuntu') @@ -103,7 +103,7 @@ class TestPuppetHandle(CiTestCase): [mock.call(('puppet', None))], mycloud.distro.install_packages.call_args_list) - @mock.patch('cloudinit.config.cc_puppet.util.subp') + @mock.patch('cloudinit.config.cc_puppet.subp.subp') def test_handler_puppet_config_installs_puppet_on_true(self, m_subp, _): """Cloud-config with 'puppet' key installs when 'install' is True.""" mycloud = self._get_cloud('ubuntu') @@ -114,7 +114,7 @@ class TestPuppetHandle(CiTestCase): [mock.call(('puppet', None))], mycloud.distro.install_packages.call_args_list) - @mock.patch('cloudinit.config.cc_puppet.util.subp') + @mock.patch('cloudinit.config.cc_puppet.subp.subp') def test_handler_puppet_config_installs_puppet_version(self, m_subp, _): """Cloud-config 'puppet' configuration can specify a version.""" mycloud = self._get_cloud('ubuntu') @@ -125,7 +125,7 @@ class TestPuppetHandle(CiTestCase): [mock.call(('puppet', '3.8'))], mycloud.distro.install_packages.call_args_list) - @mock.patch('cloudinit.config.cc_puppet.util.subp') + @mock.patch('cloudinit.config.cc_puppet.subp.subp') def test_handler_puppet_config_updates_puppet_conf(self, m_subp, m_auto): """When 'conf' is provided update values in PUPPET_CONF_PATH.""" mycloud = self._get_cloud('ubuntu') @@ -141,7 +141,7 @@ class TestPuppetHandle(CiTestCase): expected = '[agent]\nserver = puppetmaster.example.org\nother = 3\n\n' self.assertEqual(expected, content) - @mock.patch('cloudinit.config.cc_puppet.util.subp') + @mock.patch('cloudinit.config.cc_puppet.subp.subp') def test_handler_puppet_writes_csr_attributes_file(self, m_subp, m_auto): """When csr_attributes is provided creates file in PUPPET_CSR_ATTRIBUTES_PATH.""" diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py index 9ce334ac..73237d68 100644 --- a/tests/unittests/test_handler/test_handler_runcmd.py +++ b/tests/unittests/test_handler/test_handler_runcmd.py @@ -2,7 +2,7 @@ from cloudinit.config.cc_runcmd import handle, schema from cloudinit.sources import DataSourceNone -from cloudinit import (distros, helpers, cloud, util) +from cloudinit import (distros, helpers, cloud, subp, util) from cloudinit.tests.helpers import ( CiTestCase, FilesystemMockingTestCase, SchemaTestCaseMixin, skipUnlessJsonSchema) @@ -20,7 +20,7 @@ class TestRuncmd(FilesystemMockingTestCase): def setUp(self): super(TestRuncmd, self).setUp() - self.subp = util.subp + self.subp = subp.subp self.new_root = self.tmp_dir() def _get_cloud(self, distro): diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index abecc53b..85167f19 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -17,6 +17,7 @@ from io import BytesIO from cloudinit import cloud from cloudinit import distros from cloudinit import helpers +from cloudinit import subp from cloudinit import util from cloudinit.sources import DataSourceNone @@ -35,8 +36,8 @@ class TestRandomSeed(t_help.TestCase): self.unapply = [] # by default 'which' has nothing in its path - self.apply_patches([(util, 'which', self._which)]) - self.apply_patches([(util, 'subp', self._subp)]) + self.apply_patches([(subp, 'which', self._which)]) + self.apply_patches([(subp, 'subp', self._subp)]) self.subp_called = [] self.whichdata = {} diff --git a/tests/unittests/test_handler/test_handler_spacewalk.py b/tests/unittests/test_handler/test_handler_spacewalk.py index 410e6f77..26f7648f 100644 --- a/tests/unittests/test_handler/test_handler_spacewalk.py +++ b/tests/unittests/test_handler/test_handler_spacewalk.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit.config import cc_spacewalk -from cloudinit import util +from cloudinit import subp from cloudinit.tests import helpers @@ -19,20 +19,20 @@ class TestSpacewalk(helpers.TestCase): } } - @mock.patch("cloudinit.config.cc_spacewalk.util.subp") - def test_not_is_registered(self, mock_util_subp): - mock_util_subp.side_effect = util.ProcessExecutionError(exit_code=1) + @mock.patch("cloudinit.config.cc_spacewalk.subp.subp") + def test_not_is_registered(self, mock_subp): + mock_subp.side_effect = subp.ProcessExecutionError(exit_code=1) self.assertFalse(cc_spacewalk.is_registered()) - @mock.patch("cloudinit.config.cc_spacewalk.util.subp") - def test_is_registered(self, mock_util_subp): - mock_util_subp.side_effect = None + @mock.patch("cloudinit.config.cc_spacewalk.subp.subp") + def test_is_registered(self, mock_subp): + mock_subp.side_effect = None self.assertTrue(cc_spacewalk.is_registered()) - @mock.patch("cloudinit.config.cc_spacewalk.util.subp") - def test_do_register(self, mock_util_subp): + @mock.patch("cloudinit.config.cc_spacewalk.subp.subp") + def test_do_register(self, mock_subp): cc_spacewalk.do_register(**self.space_cfg['spacewalk']) - mock_util_subp.assert_called_with([ + mock_subp.assert_called_with([ 'rhnreg_ks', '--serverUrl', 'https://localhost/XMLRPC', '--profilename', 'test', diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 23626395..a56022ef 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -8,6 +8,7 @@ from cloudinit.net import ( renderers, sysconfig) from cloudinit.sources.helpers import openstack from cloudinit import temp_utils +from cloudinit import subp from cloudinit import util from cloudinit import safeyaml as yaml @@ -3192,7 +3193,7 @@ USERCTL=no def test_check_ifcfg_rh(self): """ifcfg-rh plugin is added NetworkManager.conf if conf present.""" render_dir = self.tmp_dir() - nm_cfg = util.target_path(render_dir, path=self.nm_cfg_file) + nm_cfg = subp.target_path(render_dir, path=self.nm_cfg_file) util.ensure_dir(os.path.dirname(nm_cfg)) # write a template nm.conf, note plugins is a list here @@ -3215,7 +3216,7 @@ USERCTL=no """ifcfg-rh plugin is append when plugins is a string.""" render_dir = self.tmp_path("render") os.makedirs(render_dir) - nm_cfg = util.target_path(render_dir, path=self.nm_cfg_file) + nm_cfg = subp.target_path(render_dir, path=self.nm_cfg_file) util.ensure_dir(os.path.dirname(nm_cfg)) # write a template nm.conf, note plugins is a value here @@ -3240,7 +3241,7 @@ USERCTL=no """enable_ifcfg_plugin creates plugins value if missing.""" render_dir = self.tmp_path("render") os.makedirs(render_dir) - nm_cfg = util.target_path(render_dir, path=self.nm_cfg_file) + nm_cfg = subp.target_path(render_dir, path=self.nm_cfg_file) util.ensure_dir(os.path.dirname(nm_cfg)) # write a template nm.conf, note plugins is missing @@ -3920,7 +3921,7 @@ class TestNetplanCleanDefault(CiTestCase): files = sorted(populate_dir(tmpd, content)) netplan._clean_default(target=tmpd) found = [t for t in files if os.path.exists(t)] - expected = [util.target_path(tmpd, f) for f in (astamp, anet, ayaml)] + expected = [subp.target_path(tmpd, f) for f in (astamp, anet, ayaml)] self.assertEqual(sorted(expected), found) @@ -3933,7 +3934,7 @@ class TestNetplanPostcommands(CiTestCase): @mock.patch.object(netplan.Renderer, '_netplan_generate') @mock.patch.object(netplan.Renderer, '_net_setup_link') - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_netplan_render_calls_postcmds(self, mock_subp, mock_netplan_generate, mock_net_setup_link): @@ -3947,7 +3948,7 @@ class TestNetplanPostcommands(CiTestCase): render_target = 'netplan.yaml' renderer = netplan.Renderer( {'netplan_path': render_target, 'postcmds': True}) - mock_subp.side_effect = iter([util.ProcessExecutionError]) + mock_subp.side_effect = iter([subp.ProcessExecutionError]) renderer.render_network_state(ns, target=render_dir) mock_netplan_generate.assert_called_with(run=True) @@ -3955,7 +3956,7 @@ class TestNetplanPostcommands(CiTestCase): @mock.patch('cloudinit.util.SeLinuxGuard') @mock.patch.object(netplan, "get_devicelist") - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_netplan_postcmds(self, mock_subp, mock_devlist, mock_sel): mock_sel.__enter__ = mock.Mock(return_value=False) mock_sel.__exit__ = mock.Mock() @@ -3971,7 +3972,7 @@ class TestNetplanPostcommands(CiTestCase): renderer = netplan.Renderer( {'netplan_path': render_target, 'postcmds': True}) mock_subp.side_effect = iter([ - util.ProcessExecutionError, + subp.ProcessExecutionError, ('', ''), ('', ''), ]) @@ -4260,7 +4261,7 @@ class TestNetplanRoundTrip(CiTestCase): def setUp(self): super(TestNetplanRoundTrip, self).setUp() - self.add_patch('cloudinit.net.netplan.util.subp', 'm_subp') + self.add_patch('cloudinit.net.netplan.subp.subp', 'm_subp') self.m_subp.return_value = (self.NETPLAN_INFO_OUT, '') def _render_and_read(self, network_config=None, state=None, @@ -5157,7 +5158,7 @@ def _gzip_data(data): class TestRenameInterfaces(CiTestCase): - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_rename_all(self, mock_subp): renames = [ ('00:11:22:33:44:55', 'interface0', 'virtio_net', '0x3'), @@ -5188,7 +5189,7 @@ class TestRenameInterfaces(CiTestCase): capture=True), ]) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_rename_no_driver_no_device_id(self, mock_subp): renames = [ ('00:11:22:33:44:55', 'interface0', None, None), @@ -5219,7 +5220,7 @@ class TestRenameInterfaces(CiTestCase): capture=True), ]) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_rename_all_bounce(self, mock_subp): renames = [ ('00:11:22:33:44:55', 'interface0', 'virtio_net', '0x3'), @@ -5254,7 +5255,7 @@ class TestRenameInterfaces(CiTestCase): mock.call(['ip', 'link', 'set', 'interface2', 'up'], capture=True) ]) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_rename_duplicate_macs(self, mock_subp): renames = [ ('00:11:22:33:44:55', 'eth0', 'hv_netsvc', '0x3'), @@ -5283,7 +5284,7 @@ class TestRenameInterfaces(CiTestCase): capture=True), ]) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_rename_duplicate_macs_driver_no_devid(self, mock_subp): renames = [ ('00:11:22:33:44:55', 'eth0', 'hv_netsvc', None), @@ -5312,7 +5313,7 @@ class TestRenameInterfaces(CiTestCase): capture=True), ]) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_rename_multi_mac_dups(self, mock_subp): renames = [ ('00:11:22:33:44:55', 'eth0', 'hv_netsvc', '0x3'), @@ -5351,7 +5352,7 @@ class TestRenameInterfaces(CiTestCase): capture=True), ]) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_rename_macs_case_insensitive(self, mock_subp): """_rename_interfaces must support upper or lower case macs.""" renames = [ diff --git a/tests/unittests/test_net_freebsd.py b/tests/unittests/test_net_freebsd.py index 48296c30..414b4830 100644 --- a/tests/unittests/test_net_freebsd.py +++ b/tests/unittests/test_net_freebsd.py @@ -7,7 +7,7 @@ SAMPLE_FREEBSD_IFCONFIG_OUT = readResource("netinfo/freebsd-ifconfig-output") class TestInterfacesByMac(CiTestCase): - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') @mock.patch('cloudinit.util.is_FreeBSD') def test_get_interfaces_by_mac(self, mock_is_FreeBSD, mock_subp): mock_is_FreeBSD.return_value = True diff --git a/tests/unittests/test_render_cloudcfg.py b/tests/unittests/test_render_cloudcfg.py index 8b1e6042..393a78b1 100644 --- a/tests/unittests/test_render_cloudcfg.py +++ b/tests/unittests/test_render_cloudcfg.py @@ -5,6 +5,7 @@ import sys import pytest +from cloudinit import subp from cloudinit import util # TODO(Look to align with tools.render-cloudcfg or cloudinit.distos.OSFAMILIES) @@ -20,7 +21,7 @@ class TestRenderCloudCfg: @pytest.mark.parametrize('variant', (DISTRO_VARIANTS)) def test_variant_sets_distro_in_cloud_cfg(self, variant, tmpdir): outfile = tmpdir.join('outcfg').strpath - util.subp( + subp.subp( self.cmd + ['--variant', variant, self.tmpl_path, outfile]) with open(outfile) as stream: system_cfg = util.load_yaml(stream.read()) @@ -31,7 +32,7 @@ class TestRenderCloudCfg: @pytest.mark.parametrize('variant', (DISTRO_VARIANTS)) def test_variant_sets_default_user_in_cloud_cfg(self, variant, tmpdir): outfile = tmpdir.join('outcfg').strpath - util.subp( + subp.subp( self.cmd + ['--variant', variant, self.tmpl_path, outfile]) with open(outfile) as stream: system_cfg = util.load_yaml(stream.read()) @@ -49,7 +50,7 @@ class TestRenderCloudCfg: self, variant, renderers, tmpdir ): outfile = tmpdir.join('outcfg').strpath - util.subp( + subp.subp( self.cmd + ['--variant', variant, self.tmpl_path, outfile]) with open(outfile) as stream: system_cfg = util.load_yaml(stream.read()) diff --git a/tests/unittests/test_reporting_hyperv.py b/tests/unittests/test_reporting_hyperv.py index b3e083c6..fa8f8859 100644 --- a/tests/unittests/test_reporting_hyperv.py +++ b/tests/unittests/test_reporting_hyperv.py @@ -131,7 +131,7 @@ class TextKvpReporter(CiTestCase): self.assertEqual(0, len(kvps)) @mock.patch('cloudinit.distros.uses_systemd') - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_get_boot_telemetry(self, m_subp, m_sysd): reporter = HyperVKvpReportingHandler(kvp_file_path=self.tmp_file_path) datetime_pattern = r"\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]" diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index 4cd27eed..53d3cd5a 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -6,7 +6,7 @@ import copy import logging from cloudinit.config import cc_rh_subscription -from cloudinit import util +from cloudinit import subp from cloudinit.tests.helpers import CiTestCase, mock @@ -56,7 +56,7 @@ class GoodTests(CiTestCase): ''' reg = "The system has been registered with ID:" \ " 12345678-abde-abcde-1234-1234567890abc" - m_sman_cli.side_effect = [util.ProcessExecutionError, (reg, 'bar')] + m_sman_cli.side_effect = [subp.ProcessExecutionError, (reg, 'bar')] self.handle(self.name, self.config, self.cloud_init, self.log, self.args) self.assertIn(mock.call(['identity']), m_sman_cli.call_args_list) @@ -93,7 +93,7 @@ class GoodTests(CiTestCase): reg = "The system has been registered with ID:" \ " 12345678-abde-abcde-1234-1234567890abc" m_sman_cli.side_effect = [ - util.ProcessExecutionError, + subp.ProcessExecutionError, (reg, 'bar'), ('Service level set to: self-support', ''), ('pool1\npool3\n', ''), ('pool2\n', ''), ('', ''), @@ -161,7 +161,7 @@ class TestBadInput(CiTestCase): def test_no_password(self, m_sman_cli): '''Attempt to register without the password key/value.''' - m_sman_cli.side_effect = [util.ProcessExecutionError, + m_sman_cli.side_effect = [subp.ProcessExecutionError, (self.reg, 'bar')] self.handle(self.name, self.config_no_password, self.cloud_init, self.log, self.args) @@ -169,7 +169,7 @@ class TestBadInput(CiTestCase): def test_no_org(self, m_sman_cli): '''Attempt to register without the org key/value.''' - m_sman_cli.side_effect = [util.ProcessExecutionError] + m_sman_cli.side_effect = [subp.ProcessExecutionError] self.handle(self.name, self.config_no_key, self.cloud_init, self.log, self.args) m_sman_cli.assert_called_with(['identity']) @@ -182,7 +182,7 @@ class TestBadInput(CiTestCase): def test_service_level_without_auto(self, m_sman_cli): '''Attempt to register using service-level without auto-attach key.''' - m_sman_cli.side_effect = [util.ProcessExecutionError, + m_sman_cli.side_effect = [subp.ProcessExecutionError, (self.reg, 'bar')] self.handle(self.name, self.config_service, self.cloud_init, self.log, self.args) @@ -195,7 +195,7 @@ class TestBadInput(CiTestCase): ''' Register with pools that are not in the format of a list ''' - m_sman_cli.side_effect = [util.ProcessExecutionError, + m_sman_cli.side_effect = [subp.ProcessExecutionError, (self.reg, 'bar')] self.handle(self.name, self.config_badpool, self.cloud_init, self.log, self.args) @@ -208,7 +208,7 @@ class TestBadInput(CiTestCase): ''' Register with repos that are not in the format of a list ''' - m_sman_cli.side_effect = [util.ProcessExecutionError, + m_sman_cli.side_effect = [subp.ProcessExecutionError, (self.reg, 'bar')] self.handle(self.name, self.config_badrepo, self.cloud_init, self.log, self.args) @@ -222,7 +222,7 @@ class TestBadInput(CiTestCase): ''' Attempt to register with a key that we don't know ''' - m_sman_cli.side_effect = [util.ProcessExecutionError, + m_sman_cli.side_effect = [subp.ProcessExecutionError, (self.reg, 'bar')] self.handle(self.name, self.config_badkey, self.cloud_init, self.log, self.args) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 4b439267..737dda8b 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -1,25 +1,20 @@ # This file is part of cloud-init. See LICENSE file for license information. import io -import json import logging import os import re import shutil import stat -import sys import tempfile import yaml from unittest import mock +from cloudinit import subp from cloudinit import importer, util from cloudinit.tests import helpers -BASH = util.which('bash') -BOGUS_COMMAND = 'this-is-not-expected-to-be-a-program-name' - - class FakeSelinux(object): def __init__(self, match_what): @@ -385,7 +380,7 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): self.assertEqual(expected, util.parse_mount_info('/run/lock', lines)) @mock.patch('cloudinit.util.os') - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_get_device_info_from_zpool(self, zpool_output, m_os): # mock /dev/zfs exists m_os.path.exists.return_value = True @@ -408,17 +403,17 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): self.assertIsNone(ret) @mock.patch('cloudinit.util.os') - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_get_device_info_from_zpool_handles_no_zpool(self, m_sub, m_os): """Handle case where there is no zpool command""" # mock /dev/zfs exists m_os.path.exists.return_value = True - m_sub.side_effect = util.ProcessExecutionError("No zpool cmd") + m_sub.side_effect = subp.ProcessExecutionError("No zpool cmd") ret = util.get_device_info_from_zpool('vmzroot') self.assertIsNone(ret) @mock.patch('cloudinit.util.os') - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_get_device_info_from_zpool_on_error(self, zpool_output, m_os): # mock /dev/zfs exists m_os.path.exists.return_value = True @@ -430,7 +425,7 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): ret = util.get_device_info_from_zpool('vmzroot') self.assertIsNone(ret) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_parse_mount_with_ext(self, mount_out): mount_out.return_value = ( helpers.readResource('mount_parse_ext.txt'), '') @@ -447,7 +442,7 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): ret = util.parse_mount('/not/existing/mount') self.assertIsNone(ret) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_parse_mount_with_zfs(self, mount_out): mount_out.return_value = ( helpers.readResource('mount_parse_zfs.txt'), '') @@ -513,13 +508,13 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): """ def _dmidecode_subp(cmd): if cmd[-1] != key: - raise util.ProcessExecutionError() + raise subp.ProcessExecutionError() return (content, error) self.patched_funcs.enter_context( - mock.patch.object(util, 'which', lambda _: True)) + mock.patch("cloudinit.subp.which", side_effect=lambda _: True)) self.patched_funcs.enter_context( - mock.patch.object(util, 'subp', _dmidecode_subp)) + mock.patch("cloudinit.subp.subp", side_effect=_dmidecode_subp)) def patch_mapping(self, new_mapping): self.patched_funcs.enter_context( @@ -546,10 +541,12 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): def test_dmidecode_not_used_on_arm(self): self.patch_mapping({}) + print("current =%s", subp) self._create_sysfs_parent_directory() dmi_val = 'from-dmidecode' dmi_name = 'use-dmidecode' self._configure_dmidecode_return(dmi_name, dmi_val) + print("now =%s", subp) expected = {'armel': None, 'aarch64': dmi_val, 'x86_64': dmi_val} found = {} @@ -560,6 +557,7 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): for arch in expected: m_uname.return_value = ('x-sysname', 'x-nodename', 'x-release', 'x-version', arch) + print("now2 =%s", subp) found[arch] = util.read_dmi_data(dmi_name) self.assertEqual(expected, found) @@ -570,7 +568,7 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): def test_none_returned_if_dmidecode_not_in_path(self): self.patched_funcs.enter_context( - mock.patch.object(util, 'which', lambda _: False)) + mock.patch.object(subp, 'which', lambda _: False)) self.patch_mapping({}) self.assertIsNone(util.read_dmi_data('expect-fail')) @@ -734,218 +732,6 @@ class TestReadSeeded(helpers.TestCase): self.assertEqual(found_ud, ud) -class TestSubp(helpers.CiTestCase): - allowed_subp = [BASH, 'cat', helpers.CiTestCase.SUBP_SHELL_TRUE, - BOGUS_COMMAND, sys.executable] - - stdin2err = [BASH, '-c', 'cat >&2'] - stdin2out = ['cat'] - utf8_invalid = b'ab\xaadef' - utf8_valid = b'start \xc3\xa9 end' - utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7' - printenv = [BASH, '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--'] - - def printf_cmd(self, *args): - # bash's printf supports \xaa. So does /usr/bin/printf - # but by using bash, we remove dependency on another program. - return([BASH, '-c', 'printf "$@"', 'printf'] + list(args)) - - def test_subp_handles_bytestrings(self): - """subp can run a bytestring command if shell is True.""" - tmp_file = self.tmp_path('test.out') - cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file) - (out, _err) = util.subp(cmd.encode('utf-8'), shell=True) - self.assertEqual(u'', out) - self.assertEqual(u'', _err) - self.assertEqual('HI MOM\n', util.load_file(tmp_file)) - - def test_subp_handles_strings(self): - """subp can run a string command if shell is True.""" - tmp_file = self.tmp_path('test.out') - cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file) - (out, _err) = util.subp(cmd, shell=True) - self.assertEqual(u'', out) - self.assertEqual(u'', _err) - self.assertEqual('HI MOM\n', util.load_file(tmp_file)) - - def test_subp_handles_utf8(self): - # The given bytes contain utf-8 accented characters as seen in e.g. - # the "deja dup" package in Ubuntu. - cmd = self.printf_cmd(self.utf8_valid_2) - (out, _err) = util.subp(cmd, capture=True) - self.assertEqual(out, self.utf8_valid_2.decode('utf-8')) - - def test_subp_respects_decode_false(self): - (out, err) = util.subp(self.stdin2out, capture=True, decode=False, - data=self.utf8_valid) - self.assertTrue(isinstance(out, bytes)) - self.assertTrue(isinstance(err, bytes)) - self.assertEqual(out, self.utf8_valid) - - def test_subp_decode_ignore(self): - # this executes a string that writes invalid utf-8 to stdout - (out, _err) = util.subp(self.printf_cmd('abc\\xaadef'), - capture=True, decode='ignore') - self.assertEqual(out, 'abcdef') - - def test_subp_decode_strict_valid_utf8(self): - (out, _err) = util.subp(self.stdin2out, capture=True, - decode='strict', data=self.utf8_valid) - self.assertEqual(out, self.utf8_valid.decode('utf-8')) - - def test_subp_decode_invalid_utf8_replaces(self): - (out, _err) = util.subp(self.stdin2out, capture=True, - data=self.utf8_invalid) - expected = self.utf8_invalid.decode('utf-8', 'replace') - self.assertEqual(out, expected) - - def test_subp_decode_strict_raises(self): - args = [] - kwargs = {'args': self.stdin2out, 'capture': True, - 'decode': 'strict', 'data': self.utf8_invalid} - self.assertRaises(UnicodeDecodeError, util.subp, *args, **kwargs) - - def test_subp_capture_stderr(self): - data = b'hello world' - (out, err) = util.subp(self.stdin2err, capture=True, - decode=False, data=data, - update_env={'LC_ALL': 'C'}) - self.assertEqual(err, data) - self.assertEqual(out, b'') - - def test_subp_reads_env(self): - with mock.patch.dict("os.environ", values={'FOO': 'BAR'}): - out, _err = util.subp(self.printenv + ['FOO'], capture=True) - self.assertEqual('FOO=BAR', out.splitlines()[0]) - - def test_subp_env_and_update_env(self): - out, _err = util.subp( - self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True, - env={'FOO': 'BAR'}, - update_env={'HOME': '/myhome', 'K2': 'V2'}) - self.assertEqual( - ['FOO=BAR', 'HOME=/myhome', 'K1=', 'K2=V2'], out.splitlines()) - - def test_subp_update_env(self): - extra = {'FOO': 'BAR', 'HOME': '/root', 'K1': 'V1'} - with mock.patch.dict("os.environ", values=extra): - out, _err = util.subp( - self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True, - update_env={'HOME': '/myhome', 'K2': 'V2'}) - - self.assertEqual( - ['FOO=BAR', 'HOME=/myhome', 'K1=V1', 'K2=V2'], out.splitlines()) - - def test_subp_warn_missing_shebang(self): - """Warn on no #! in script""" - noshebang = self.tmp_path('noshebang') - util.write_file(noshebang, 'true\n') - - os.chmod(noshebang, os.stat(noshebang).st_mode | stat.S_IEXEC) - with self.allow_subp([noshebang]): - self.assertRaisesRegex(util.ProcessExecutionError, - r'Missing #! in script\?', - util.subp, (noshebang,)) - - def test_subp_combined_stderr_stdout(self): - """Providing combine_capture as True redirects stderr to stdout.""" - data = b'hello world' - (out, err) = util.subp(self.stdin2err, capture=True, - combine_capture=True, decode=False, data=data) - self.assertEqual(b'', err) - self.assertEqual(data, out) - - def test_returns_none_if_no_capture(self): - (out, err) = util.subp(self.stdin2out, data=b'', capture=False) - self.assertIsNone(err) - self.assertIsNone(out) - - def test_exception_has_out_err_are_bytes_if_decode_false(self): - """Raised exc should have stderr, stdout as bytes if no decode.""" - with self.assertRaises(util.ProcessExecutionError) as cm: - util.subp([BOGUS_COMMAND], decode=False) - self.assertTrue(isinstance(cm.exception.stdout, bytes)) - self.assertTrue(isinstance(cm.exception.stderr, bytes)) - - def test_exception_has_out_err_are_bytes_if_decode_true(self): - """Raised exc should have stderr, stdout as string if no decode.""" - with self.assertRaises(util.ProcessExecutionError) as cm: - util.subp([BOGUS_COMMAND], decode=True) - self.assertTrue(isinstance(cm.exception.stdout, str)) - self.assertTrue(isinstance(cm.exception.stderr, str)) - - def test_bunch_of_slashes_in_path(self): - self.assertEqual("/target/my/path/", - util.target_path("/target/", "//my/path/")) - self.assertEqual("/target/my/path/", - util.target_path("/target/", "///my/path/")) - - def test_c_lang_can_take_utf8_args(self): - """Independent of system LC_CTYPE, args can contain utf-8 strings. - - When python starts up, its default encoding gets set based on - the value of LC_CTYPE. If no system locale is set, the default - encoding for both python2 and python3 in some paths will end up - being ascii. - - Attempts to use setlocale or patching (or changing) os.environ - in the current environment seem to not be effective. - - This test starts up a python with LC_CTYPE set to C so that - the default encoding will be set to ascii. In such an environment - Popen(['command', 'non-ascii-arg']) would cause a UnicodeDecodeError. - """ - python_prog = '\n'.join([ - 'import json, sys', - 'from cloudinit.util import subp', - 'data = sys.stdin.read()', - 'cmd = json.loads(data)', - 'subp(cmd, capture=False)', - '']) - cmd = [BASH, '-c', 'echo -n "$@"', '--', - self.utf8_valid.decode("utf-8")] - python_subp = [sys.executable, '-c', python_prog] - - out, _err = util.subp( - python_subp, update_env={'LC_CTYPE': 'C'}, - data=json.dumps(cmd).encode("utf-8"), - decode=False) - self.assertEqual(self.utf8_valid, out) - - def test_bogus_command_logs_status_messages(self): - """status_cb gets status messages logs on bogus commands provided.""" - logs = [] - - def status_cb(log): - logs.append(log) - - with self.assertRaises(util.ProcessExecutionError): - util.subp([BOGUS_COMMAND], status_cb=status_cb) - - expected = [ - 'Begin run command: {cmd}\n'.format(cmd=BOGUS_COMMAND), - 'ERROR: End run command: invalid command provided\n'] - self.assertEqual(expected, logs) - - def test_command_logs_exit_codes_to_status_cb(self): - """status_cb gets status messages containing command exit code.""" - logs = [] - - def status_cb(log): - logs.append(log) - - with self.assertRaises(util.ProcessExecutionError): - util.subp([BASH, '-c', 'exit 2'], status_cb=status_cb) - util.subp([BASH, '-c', 'exit 0'], status_cb=status_cb) - - expected = [ - 'Begin run command: %s -c exit 2\n' % BASH, - 'ERROR: End run command: exit(2)\n', - 'Begin run command: %s -c exit 0\n' % BASH, - 'End run command: exit(0)\n'] - self.assertEqual(expected, logs) - - class TestEncode(helpers.TestCase): """Test the encoding functions""" def test_decode_binary_plain_text_with_hex(self): @@ -966,7 +752,7 @@ class TestProcessExecutionError(helpers.TestCase): empty_description = 'Unexpected error while running command.' def test_pexec_error_indent_text(self): - error = util.ProcessExecutionError() + error = subp.ProcessExecutionError() msg = 'abc\ndef' formatted = 'abc\n{0}def'.format(' ' * 4) self.assertEqual(error._indent_text(msg, indent_level=4), formatted) @@ -976,10 +762,10 @@ class TestProcessExecutionError(helpers.TestCase): error._indent_text(msg.encode()), type(msg.encode())) def test_pexec_error_type(self): - self.assertIsInstance(util.ProcessExecutionError(), IOError) + self.assertIsInstance(subp.ProcessExecutionError(), IOError) def test_pexec_error_empty_msgs(self): - error = util.ProcessExecutionError() + error = subp.ProcessExecutionError() self.assertTrue(all(attr == self.empty_attr for attr in (error.stderr, error.stdout, error.reason))) self.assertEqual(error.description, self.empty_description) @@ -993,7 +779,7 @@ class TestProcessExecutionError(helpers.TestCase): stderr_msg = 'error error' cmd = 'test command' exit_code = 3 - error = util.ProcessExecutionError( + error = subp.ProcessExecutionError( stdout=stdout_msg, stderr=stderr_msg, exit_code=3, cmd=cmd) self.assertEqual(str(error), self.template.format( description=self.empty_description, stdout=stdout_msg, @@ -1004,7 +790,7 @@ class TestProcessExecutionError(helpers.TestCase): # make sure bytes is converted handled properly when formatting stdout_msg = 'multi\nline\noutput message'.encode() stderr_msg = 'multi\nline\nerror message\n\n\n' - error = util.ProcessExecutionError( + error = subp.ProcessExecutionError( stdout=stdout_msg, stderr=stderr_msg) self.assertEqual( str(error), @@ -1170,7 +956,7 @@ class TestGetProcEnv(helpers.TestCase): self.assertEqual(my_ppid, util.get_proc_ppid(my_pid)) -@mock.patch('cloudinit.util.subp') +@mock.patch('cloudinit.subp.subp') def test_find_devs_with_openbsd(m_subp): m_subp.return_value = ( 'cd0:,sd0:630d98d32b5d3759,sd1:,fd0:', '' @@ -1179,7 +965,7 @@ def test_find_devs_with_openbsd(m_subp): assert devlist == ['/dev/cd0a', '/dev/sd1i'] -@mock.patch('cloudinit.util.subp') +@mock.patch('cloudinit.subp.subp') def test_find_devs_with_openbsd_with_criteria(m_subp): m_subp.return_value = ( 'cd0:,sd0:630d98d32b5d3759,sd1:,fd0:', '' @@ -1209,7 +995,7 @@ def test_find_devs_with_freebsd(m_glob): assert devlist == ['/dev/msdosfs/EFISYS'] -@mock.patch("cloudinit.util.subp") +@mock.patch("cloudinit.subp.subp") def test_find_devs_with_netbsd(m_subp): side_effect_values = [ ("ld0 dk0 dk1 cd0", ""), diff --git a/tests/unittests/test_vmware/test_guestcust_util.py b/tests/unittests/test_vmware/test_guestcust_util.py index 394bee9f..c8b59d83 100644 --- a/tests/unittests/test_vmware/test_guestcust_util.py +++ b/tests/unittests/test_vmware/test_guestcust_util.py @@ -5,7 +5,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import util +from cloudinit import subp from cloudinit.sources.helpers.vmware.imc.config import Config from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile from cloudinit.sources.helpers.vmware.imc.guestcust_util import ( @@ -21,7 +21,7 @@ class TestGuestCustUtil(CiTestCase): This test is designed to verify the behavior if vmware-toolbox-cmd is not installed. """ - with mock.patch.object(util, 'which', return_value=None): + with mock.patch.object(subp, 'which', return_value=None): self.assertEqual( get_tools_config('section', 'key', 'defaultVal'), 'defaultVal') @@ -30,10 +30,10 @@ class TestGuestCustUtil(CiTestCase): This test is designed to verify the behavior if internal exception is raised. """ - with mock.patch.object(util, 'which', return_value='/dummy/path'): - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'which', return_value='/dummy/path'): + with mock.patch.object(subp, 'subp', return_value=('key=value', b''), - side_effect=util.ProcessExecutionError( + side_effect=subp.ProcessExecutionError( "subp failed", exit_code=99)): # verify return value is 'defaultVal', not 'value'. self.assertEqual( @@ -45,28 +45,28 @@ class TestGuestCustUtil(CiTestCase): This test is designed to verify the value could be parsed from key = value of the given [section] """ - with mock.patch.object(util, 'which', return_value='/dummy/path'): + with mock.patch.object(subp, 'which', return_value='/dummy/path'): # value is not blank - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('key = value ', b'')): self.assertEqual( get_tools_config('section', 'key', 'defaultVal'), 'value') # value is blank - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('key = ', b'')): self.assertEqual( get_tools_config('section', 'key', 'defaultVal'), '') # value contains = - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('key=Bar=Wark', b'')): self.assertEqual( get_tools_config('section', 'key', 'defaultVal'), 'Bar=Wark') # value contains specific characters - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('[a] b.c_d=e-f', b'')): self.assertEqual( get_tools_config('section', 'key', 'defaultVal'), @@ -87,7 +87,7 @@ class TestGuestCustUtil(CiTestCase): # post gc status is YES, subp is called to execute command cf._insertKey("MISC|POST-GC-STATUS", "YES") conf = Config(cf) - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('ok', b'')) as mockobj: self.assertEqual( set_gc_status(conf, 'Successful'), ('ok', b'')) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index c3113705..01916b70 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -4,5 +4,6 @@ dhensby lucasmoura matthewruffell nishigori +smoser tomponline TheRealFalcon -- cgit v1.2.3 From 0755cff078d5931e1d8e151bdcb84afb92bc0f02 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 22 Jun 2020 10:30:05 -0400 Subject: cc_final_message: don't create directories when writing boot-finished (#445) If the instance symlink doesn't exist, then we shouldn't create a directory in its place, because that breaks future boots. LP: #1883903 --- cloudinit/config/cc_final_message.py | 2 +- cloudinit/config/tests/test_final_message.py | 33 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 cloudinit/config/tests/test_final_message.py (limited to 'cloudinit/config/tests') diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index fd141541..3441f7a9 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -78,7 +78,7 @@ def handle(_name, cfg, cloud, log, args): boot_fin_fn = cloud.paths.boot_finished try: contents = "%s - %s - v. %s\n" % (uptime, ts, cver) - util.write_file(boot_fin_fn, contents) + util.write_file(boot_fin_fn, contents, ensure_dir_exists=False) except Exception: util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn) diff --git a/cloudinit/config/tests/test_final_message.py b/cloudinit/config/tests/test_final_message.py new file mode 100644 index 00000000..76cb0ad1 --- /dev/null +++ b/cloudinit/config/tests/test_final_message.py @@ -0,0 +1,33 @@ +# This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + +import pytest + +from cloudinit.config.cc_final_message import handle + + +class TestHandle: + # TODO: Expand these tests to cover full functionality; currently they only + # cover the logic around how the boot-finished file is written (and not its + # contents). + + @pytest.mark.parametrize( + "instance_dir_exists,file_is_written", [(True, True), (False, False)] + ) + def test_boot_finished_written( + self, instance_dir_exists, file_is_written, tmpdir + ): + instance_dir = tmpdir.join("var/lib/cloud/instance") + if instance_dir_exists: + instance_dir.ensure_dir() + boot_finished = instance_dir.join("boot-finished") + + m_cloud = mock.Mock( + paths=mock.Mock(boot_finished=boot_finished.strpath) + ) + + handle(None, {}, m_cloud, mock.Mock(), []) + + # We should not change the status of the instance directory + assert instance_dir_exists == instance_dir.exists() + assert file_is_written == boot_finished.exists() -- cgit v1.2.3 From 66e114a660c53400e389f119781f378311b65108 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 30 Jun 2020 13:17:40 -0400 Subject: Enable use of the caplog fixture in pytest tests, and add a cc_final_message test using it (#461) caplog is only available in pytest itself from 3.0 onwards. In xenial, we only have pytest 2.8.7. However, in xenial we do have pytest-catchlog available (as python3-pytest-catchlog), so we use that where appropriate. --- .travis.yml | 2 +- HACKING.rst | 9 +++++---- cloudinit/config/tests/test_final_message.py | 19 ++++++++++++++++--- packages/bddeb | 2 ++ tox.ini | 1 + 5 files changed, 25 insertions(+), 8 deletions(-) (limited to 'cloudinit/config/tests') diff --git a/.travis.yml b/.travis.yml index 95c591fd..9844c065 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,7 +72,7 @@ matrix: - cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc script: # Ubuntu LTS: Build - - ./packages/bddeb -S -d + - ./packages/bddeb -S -d --release xenial - | needs_caching=false if [ -e "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" ]; then diff --git a/HACKING.rst b/HACKING.rst index 1e8aca7a..27a38bc3 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -218,11 +218,12 @@ The following guidelines should be followed: [#fixture-list]_: * ``cache`` - * ``capsys`` * ``capfd`` - * ``record_xml_property`` + * ``caplog`` (provided by ``python3-pytest-catchlog`` on xenial) + * ``capsys`` * ``monkeypatch`` * ``pytestconfig`` + * ``record_xml_property`` * ``recwarn`` * ``tmpdir_factory`` * ``tmpdir`` @@ -328,9 +329,9 @@ variable annotations specified in `PEP-526`_ were introduced in Python .. [#fixture-list] This list of fixtures (with markup) can be reproduced by running:: - py.test-3 --fixtures -q | grep "^[^ ]" | grep -v no | sed 's/.*/* ``\0``/' + py.test-3 --fixtures -q | grep "^[^ -]" | grep -v '\(no\|capturelog\)' | sort | sed 's/.*/* ``\0``/' - in a xenial lxd container with python3-pytest installed. + in a xenial lxd container with python3-pytest-catchlog installed. Feature Flags ------------- diff --git a/cloudinit/config/tests/test_final_message.py b/cloudinit/config/tests/test_final_message.py index 76cb0ad1..46ba99b2 100644 --- a/cloudinit/config/tests/test_final_message.py +++ b/cloudinit/config/tests/test_final_message.py @@ -1,4 +1,5 @@ # This file is part of cloud-init. See LICENSE file for license information. +import logging from unittest import mock import pytest @@ -12,10 +13,19 @@ class TestHandle: # contents). @pytest.mark.parametrize( - "instance_dir_exists,file_is_written", [(True, True), (False, False)] + "instance_dir_exists,file_is_written,expected_log_substring", + [ + (True, True, None), + (False, False, "Failed to write boot finished file "), + ], ) def test_boot_finished_written( - self, instance_dir_exists, file_is_written, tmpdir + self, + instance_dir_exists, + file_is_written, + expected_log_substring, + caplog, + tmpdir, ): instance_dir = tmpdir.join("var/lib/cloud/instance") if instance_dir_exists: @@ -26,8 +36,11 @@ class TestHandle: paths=mock.Mock(boot_finished=boot_finished.strpath) ) - handle(None, {}, m_cloud, mock.Mock(), []) + handle(None, {}, m_cloud, logging.getLogger(), []) # We should not change the status of the instance directory assert instance_dir_exists == instance_dir.exists() assert file_is_written == boot_finished.exists() + + if expected_log_substring: + assert expected_log_substring in caplog.text diff --git a/packages/bddeb b/packages/bddeb index 78b1c83b..b0f219b6 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -91,6 +91,8 @@ def write_debian_folder(root, templ_data, cloud_util_deps): # NOTE: python package was moved to the front after debuild -S would fail with # 'Please add apropriate interpreter' errors (as in debian bug 861132) requires.extend(['python3'] + reqs + test_reqs) + if templ_data['debian_release'] == 'xenial': + requires.append('python3-pytest-catchlog') templater.render_to_file(util.abs_join(find_root(), 'packages', 'debian', 'control.in'), util.abs_join(deb_dir, 'control'), diff --git a/tox.ini b/tox.ini index ebcebc41..847a2fbd 100644 --- a/tox.ini +++ b/tox.ini @@ -79,6 +79,7 @@ deps = # test-requirements httpretty==0.9.6 mock==1.3.0 + pytest-catchlog==1.2.1 [testenv:xenial] # When updating this commands definition, also update the definition in -- cgit v1.2.3