From d5dbbd1c54844a41067c673e915ada629fb4cd8f Mon Sep 17 00:00:00 2001 From: Daniel Hensby Date: Wed, 11 Mar 2020 14:55:30 +0000 Subject: Add pub_key_ed25519 to cc_phone_home (#237) --- cloudinit/config/cc_phone_home.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index b8e27090..733c3910 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -19,6 +19,7 @@ keys to post. Available keys are: - ``pub_key_dsa`` - ``pub_key_rsa`` - ``pub_key_ecdsa`` + - ``pub_key_ed25519`` - ``instance_id`` - ``hostname`` - ``fdqn`` @@ -52,6 +53,7 @@ POST_LIST_ALL = [ 'pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', + 'pub_key_ed25519', 'instance_id', 'hostname', 'fqdn' @@ -105,6 +107,7 @@ def handle(name, cfg, cloud, log, args): 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', + 'pub_key_ed25519': '/etc/ssh/ssh_host_ed25519_key.pub', } for (n, path) in pubkeys.items(): -- cgit v1.2.3 From 1426a0d036a7f6707aac142a51f155f507d26739 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 13 Mar 2020 15:49:42 -0400 Subject: cc_resolv_conf: introduce tests and stabilise output across Python versions (#251) This is a follow-up to #144 which fixed the rendering behaviour. While writing the tests, CI failed due to dict iteration differences across Python versions, so this also sorts output so that we will produce the same output across Python versions. --- cloudinit/config/tests/test_resolv_conf.py | 86 ++++++++++++++++++++++++++++++ templates/resolv.conf.tmpl | 2 +- 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 cloudinit/config/tests/test_resolv_conf.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/tests/test_resolv_conf.py b/cloudinit/config/tests/test_resolv_conf.py new file mode 100644 index 00000000..f3d97b82 --- /dev/null +++ b/cloudinit/config/tests/test_resolv_conf.py @@ -0,0 +1,86 @@ +from unittest import mock + +import pytest + +from cloudinit.config.cc_resolv_conf import generate_resolv_conf + + +EXPECTED_HEADER = """\ +# Your system has been configured with 'manage-resolv-conf' set to true. +# As a result, cloud-init has written this file with configuration data +# that it has been provided. Cloud-init, by default, will write this file +# a single time (PER_ONCE). +#\n\n""" + + +class TestGenerateResolvConf: + @mock.patch("cloudinit.config.cc_resolv_conf.templater.render_to_file") + def test_default_target_fname_is_etc_resolvconf(self, m_render_to_file): + generate_resolv_conf("templates/resolv.conf.tmpl", mock.MagicMock()) + + assert [ + mock.call(mock.ANY, "/etc/resolv.conf", mock.ANY) + ] == m_render_to_file.call_args_list + + @mock.patch("cloudinit.config.cc_resolv_conf.templater.render_to_file") + def test_target_fname_is_used_if_passed(self, m_render_to_file): + generate_resolv_conf( + "templates/resolv.conf.tmpl", mock.MagicMock(), "/use/this/path" + ) + + assert [ + mock.call(mock.ANY, "/use/this/path", mock.ANY) + ] == m_render_to_file.call_args_list + + # Patch in templater so we can assert on the actual generated content + @mock.patch("cloudinit.templater.util.write_file") + # Parameterise with the value to be passed to generate_resolv_conf as the + # `params` parameter, and a list of the expected lines after the header as + # `extra_lines`. + @pytest.mark.parametrize( + "params,expected_extra_line", + [ + # No options + ({}, None), + # Just a true flag + ({"options": {"foo": True}}, "options foo"), + # Just a false flag + ({"options": {"foo": False}}, None), + # Just an option + ({"options": {"foo": "some_value"}}, "options foo:some_value"), + # A true flag and an option + ( + {"options": {"foo": "some_value", "bar": True}}, + "options bar foo:some_value", + ), + # Two options + ( + {"options": {"foo": "some_value", "bar": "other_value"}}, + "options bar:other_value foo:some_value", + ), + # Everything + ( + { + "options": { + "foo": "some_value", + "bar": "other_value", + "baz": False, + "spam": True, + } + }, + "options spam bar:other_value foo:some_value", + ), + ], + ) + def test_flags_and_options( + self, m_write_file, params, expected_extra_line + ): + generate_resolv_conf("templates/resolv.conf.tmpl", params) + + expected_content = EXPECTED_HEADER + if expected_extra_line is not None: + # If we have any extra lines, expect a trailing newline + expected_content += "\n".join([expected_extra_line, ""]) + assert [ + mock.call(mock.ANY, expected_content, mode=mock.ANY) + ] == m_write_file.call_args_list diff --git a/templates/resolv.conf.tmpl b/templates/resolv.conf.tmpl index b6d46b4b..f870be67 100644 --- a/templates/resolv.conf.tmpl +++ b/templates/resolv.conf.tmpl @@ -32,7 +32,7 @@ options {{flag-}} {% endfor %} -{%- for key, value in options.items() %} +{%- for key, value in options.items()|sort %} {{key}}:{{value-}} {% endfor %} {% endif %} -- cgit v1.2.3 From ade47866aa2f81ab0f3baabbb1fc1e484abdb741 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 19 Mar 2020 12:01:15 -0400 Subject: cloudinit/tests: remove unneeded with_logs configuration (#263) These classes don't use `self.logs` anywhere in their body, so we can remove the `with_logs = True` setting from them. These instances were found using astpath[0], with the following invocation: astpath "//Name[@id='with_logs' and not(ancestor::ClassDef//Attribute[@attr='logs'])]" [0] https://github.com/hchasestevens/astpath --- cloudinit/cmd/tests/test_main.py | 2 -- cloudinit/config/tests/test_disable_ec2_metadata.py | 2 -- cloudinit/net/tests/test_init.py | 6 ------ cloudinit/sources/tests/test_oracle.py | 2 -- tests/unittests/test_datasource/test_azure.py | 2 -- tests/unittests/test_datasource/test_maas.py | 1 - tests/unittests/test_handler/test_handler_locale.py | 2 -- tests/unittests/test_handler/test_handler_puppet.py | 2 -- tests/unittests/test_util.py | 1 - 9 files changed, 20 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/cmd/tests/test_main.py b/cloudinit/cmd/tests/test_main.py index 384fddc6..585b3b0e 100644 --- a/cloudinit/cmd/tests/test_main.py +++ b/cloudinit/cmd/tests/test_main.py @@ -18,8 +18,6 @@ myargs = namedtuple('MyArgs', 'debug files force local reporter subcommand') class TestMain(FilesystemMockingTestCase): - with_logs = True - def setUp(self): super(TestMain, self).setUp() self.new_root = self.tmp_dir() diff --git a/cloudinit/config/tests/test_disable_ec2_metadata.py b/cloudinit/config/tests/test_disable_ec2_metadata.py index 67646b03..823917c7 100644 --- a/cloudinit/config/tests/test_disable_ec2_metadata.py +++ b/cloudinit/config/tests/test_disable_ec2_metadata.py @@ -15,8 +15,6 @@ DISABLE_CFG = {'disable_ec2_metadata': 'true'} class TestEC2MetadataRoute(CiTestCase): - with_logs = True - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') def test_disable_ifconfig(self, m_subp, m_which): diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index 5081a337..a965699a 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -341,8 +341,6 @@ class TestGenerateFallbackConfig(CiTestCase): class TestNetFindFallBackNic(CiTestCase): - with_logs = True - def setUp(self): super(TestNetFindFallBackNic, self).setUp() sys_mock = mock.patch('cloudinit.net.get_sys_class_path') @@ -995,8 +993,6 @@ class TestExtractPhysdevs(CiTestCase): class TestWaitForPhysdevs(CiTestCase): - with_logs = True - def setUp(self): super(TestWaitForPhysdevs, self).setUp() self.add_patch('cloudinit.net.get_interfaces_by_mac', @@ -1071,8 +1067,6 @@ class TestWaitForPhysdevs(CiTestCase): class TestNetFailOver(CiTestCase): - with_logs = True - def setUp(self): super(TestNetFailOver, self).setUp() self.add_patch('cloudinit.net.util', 'm_util') diff --git a/cloudinit/sources/tests/test_oracle.py b/cloudinit/sources/tests/test_oracle.py index abf3d359..316efc4f 100644 --- a/cloudinit/sources/tests/test_oracle.py +++ b/cloudinit/sources/tests/test_oracle.py @@ -588,8 +588,6 @@ class TestNetworkConfigFromOpcImds(test_helpers.CiTestCase): class TestNetworkConfigFiltersNetFailover(test_helpers.CiTestCase): - with_logs = True - def setUp(self): super(TestNetworkConfigFiltersNetFailover, self).setUp() self.add_patch(DS_PATH + '.get_interfaces_by_mac', diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index a809fd87..2c6aa44d 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -383,8 +383,6 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): class TestAzureDataSource(CiTestCase): - with_logs = True - def setUp(self): super(TestAzureDataSource, self).setUp() self.tmp = self.tmp_dir() diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 2a81d3f5..41b6c27b 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -158,7 +158,6 @@ class TestMAASDataSource(CiTestCase): @mock.patch("cloudinit.sources.DataSourceMAAS.url_helper.OauthUrlHelper") class TestGetOauthHelper(CiTestCase): - with_logs = True base_cfg = {'consumer_key': 'FAKE_CONSUMER_KEY', 'token_key': 'FAKE_TOKEN_KEY', 'token_secret': 'FAKE_TOKEN_SECRET', diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index 2b22559f..407aa6c4 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -29,8 +29,6 @@ LOG = logging.getLogger(__name__) class TestLocale(t_help.FilesystemMockingTestCase): - with_logs = True - def setUp(self): super(TestLocale, self).setUp() self.new_root = tempfile.mkdtemp() diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py index 1494177d..04aa7d03 100644 --- a/tests/unittests/test_handler/test_handler_puppet.py +++ b/tests/unittests/test_handler/test_handler_puppet.py @@ -16,8 +16,6 @@ LOG = logging.getLogger(__name__) @mock.patch('cloudinit.config.cc_puppet.os') class TestAutostartPuppet(CiTestCase): - with_logs = True - def test_wb_autostart_puppet_updates_puppet_default(self, m_os, m_util): """Update /etc/default/puppet to autostart if it exists.""" diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 9ff17f52..5b2eaa69 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -737,7 +737,6 @@ class TestReadSeeded(helpers.TestCase): class TestSubp(helpers.CiTestCase): - with_logs = True allowed_subp = [BASH, 'cat', helpers.CiTestCase.SUBP_SHELL_TRUE, BOGUS_COMMAND, sys.executable] -- cgit v1.2.3 From a18338e99d0ab91f1f08b969bce8181938a284cc Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 23 Mar 2020 15:39:51 -0400 Subject: cc_apt_configure/util: combine search_for_mirror implementations (#271) These two implementations had drifted away from one another very slightly. Reconcile them and then remove the one in cc_apt_configure. --- cloudinit/config/cc_apt_configure.py | 23 ++-------------------- cloudinit/util.py | 5 +++++ .../test_handler/test_handler_apt_source_v3.py | 6 +++--- 3 files changed, 10 insertions(+), 24 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index c44dec45..ff58c062 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -763,25 +763,6 @@ def convert_to_v3_apt_format(cfg): return cfg -def search_for_mirror(candidates): - """ - Search through a list of mirror urls for one that works - This needs to return quickly. - """ - if candidates is None: - return None - - LOG.debug("search for mirror in candidates: '%s'", candidates) - for cand in candidates: - try: - if util.is_resolvable_url(cand): - LOG.debug("found working mirror: '%s'", cand) - return cand - except Exception: - pass - return None - - def search_for_mirror_dns(configured, mirrortype, cfg, cloud): """ Try to resolve a list of predefines DNS names to pick mirrors @@ -813,7 +794,7 @@ def search_for_mirror_dns(configured, mirrortype, cfg, cloud): for post in doms: mirror_list.append(mirrorfmt % (post)) - mirror = search_for_mirror(mirror_list) + mirror = util.search_for_mirror(mirror_list) return mirror @@ -876,7 +857,7 @@ def get_mirror(cfg, mirrortype, arch, cloud): # fallback to search if specified if mirror is None: # list of mirrors to try to resolve - mirror = search_for_mirror(mcfg.get("search", None)) + mirror = util.search_for_mirror(mcfg.get("search", None)) # fallback to search_dns if specified if mirror is None: diff --git a/cloudinit/util.py b/cloudinit/util.py index 6a641550..9cc87d7d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1230,9 +1230,14 @@ def search_for_mirror(candidates): Search through a list of mirror urls for one that works This needs to return quickly. """ + if candidates is None: + return None + + LOG.debug("search for mirror in candidates: '%s'", candidates) for cand in candidates: try: if is_resolvable_url(cand): + LOG.debug("found working mirror: '%s'", cand) return cand except Exception: pass diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py index 90949b6d..4762dbef 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -670,7 +670,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): "security": [{'arches': ["default"], "search": ["sfailme", smir]}]} - with mock.patch.object(cc_apt_configure, 'search_for_mirror', + with mock.patch.object(cc_apt_configure.util, 'search_for_mirror', side_effect=[pmir, smir]) as mocksearch: mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, 'amd64') @@ -709,7 +709,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): mockgm.assert_has_calls(calls) # should not be called, since primary is specified - with mock.patch.object(cc_apt_configure, + with mock.patch.object(cc_apt_configure.util, 'search_for_mirror') as mockse: mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch) mockse.assert_not_called() @@ -974,7 +974,7 @@ deb http://ubuntu.com/ubuntu/ xenial-proposed main""") mocksdns.assert_has_calls(calls) # first return is for the non-dns call before - with mock.patch.object(cc_apt_configure, 'search_for_mirror', + with mock.patch.object(cc_apt_configure.util, 'search_for_mirror', side_effect=[None, pmir, None, smir]) as mockse: mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch) -- cgit v1.2.3 From 04771d75a8670f07ae4c75b5892e3b795e9d1a07 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 23 Mar 2020 17:22:42 -0400 Subject: cc_disk_setup: fix RuntimeError (#270) Addresses "Runtime Error: dictionary keys changed during iteration". Co-authored-by: Noah Meyerhans LP: #1868327 --- cloudinit/config/cc_disk_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 0796cb7b..45925755 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -163,7 +163,7 @@ def handle(_name, cfg, cloud, log, _args): def update_disk_setup_devices(disk_setup, tformer): # update 'disk_setup' dictionary anywhere were a device may occur # update it with the response from 'tformer' - for origname in disk_setup.keys(): + for origname in list(disk_setup): transformed = tformer(origname) if transformed is None or transformed == origname: continue -- cgit v1.2.3 From 993f3e3e76e56266a83776a8f54dbb3ba59cfce7 Mon Sep 17 00:00:00 2001 From: GonĂ©ri Le Bouder Date: Wed, 25 Mar 2020 13:44:10 -0400 Subject: set_passwords: avoid chpasswd on BSD (#268) Avoid chpasswd on all the BSD variants. --- cloudinit/config/cc_set_passwords.py | 2 +- cloudinit/config/tests/test_set_passwords.py | 16 ++++++++-------- cloudinit/tests/test_util.py | 6 ++++-- cloudinit/util.py | 7 ++++++- 4 files changed, 19 insertions(+), 12 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 4943d545..7b7aa885 100755 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -241,7 +241,7 @@ def rand_user_password(pwlen=20): def chpasswd(distro, plist_in, hashed=False): - if util.is_FreeBSD(): + if util.is_BSD(): for pentry in plist_in.splitlines(): u, p = pentry.split(":") distro.set_passwd(u, p, hashed=hashed) diff --git a/cloudinit/config/tests/test_set_passwords.py b/cloudinit/config/tests/test_set_passwords.py index 8247c388..2732bd60 100644 --- a/cloudinit/config/tests/test_set_passwords.py +++ b/cloudinit/config/tests/test_set_passwords.py @@ -112,12 +112,12 @@ class TestSetPasswordsHandle(CiTestCase): '\n'.join(valid_hashed_pwds) + '\n')], m_subp.call_args_list) - @mock.patch(MODPATH + "util.is_FreeBSD") + @mock.patch(MODPATH + "util.is_BSD") @mock.patch(MODPATH + "util.subp") - def test_freebsd_calls_custom_pw_cmds_to_set_and_expire_passwords( - self, m_subp, m_is_freebsd): - """FreeBSD calls custom pw commands instead of chpasswd and passwd""" - m_is_freebsd.return_value = True + def test_bsd_calls_custom_pw_cmds_to_set_and_expire_passwords( + self, m_subp, m_is_bsd): + """BSD don't use chpasswd""" + m_is_bsd.return_value = True cloud = self.tmp_cloud(distro='freebsd') valid_pwds = ['ubuntu:passw0rd'] cfg = {'chpasswd': {'list': valid_pwds}} @@ -129,12 +129,12 @@ class TestSetPasswordsHandle(CiTestCase): mock.call(['pw', 'usermod', 'ubuntu', '-p', '01-Jan-1970'])], m_subp.call_args_list) - @mock.patch(MODPATH + "util.is_FreeBSD") + @mock.patch(MODPATH + "util.is_BSD") @mock.patch(MODPATH + "util.subp") def test_handle_on_chpasswd_list_creates_random_passwords(self, m_subp, - m_is_freebsd): + m_is_bsd): """handle parses command set random passwords.""" - m_is_freebsd.return_value = False + m_is_bsd.return_value = False cloud = self.tmp_cloud(distro='ubuntu') valid_random_pwds = [ 'root:R', diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index 877ab5c5..bfccfe1e 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -441,13 +441,15 @@ class TestGetLinuxDistro(CiTestCase): @mock.patch('platform.system') @mock.patch('platform.release') @mock.patch('cloudinit.util._parse_redhat_release') - def test_get_linux_freebsd(self, m_path_exists, m_platform_release, - m_platform_system, m_parse_redhat_release): + def test_get_linux_freebsd(self, m_parse_redhat_release, + m_platform_release, + m_platform_system, m_path_exists): """Verify we get the correct name and release name on FreeBSD.""" m_path_exists.return_value = False m_platform_release.return_value = '12.0-RELEASE-p10' m_platform_system.return_value = 'FreeBSD' m_parse_redhat_release.return_value = {} + util.is_BSD.cache_clear() dist = util.get_linux_distro() self.assertEqual(('freebsd', '12.0-RELEASE-p10', ''), dist) diff --git a/cloudinit/util.py b/cloudinit/util.py index 89889459..db60b9d2 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -542,6 +542,11 @@ def is_ipv4(instr): return len(toks) == 4 +@lru_cache() +def is_BSD(): + return 'BSD' in platform.system() + + @lru_cache() def is_FreeBSD(): return system_info()['variant'] == "freebsd" @@ -625,7 +630,7 @@ def get_linux_distro(): flavor = match.groupdict()['codename'] if distro_name == 'rhel': distro_name = 'redhat' - elif 'BSD' in platform.system(): + elif is_BSD(): distro_name = platform.system().lower() distro_version = platform.release() else: -- cgit v1.2.3 From 723e2bc1ad14687a8d6846b9ca73620e4e4090b1 Mon Sep 17 00:00:00 2001 From: Andrew Beresford Date: Thu, 2 Apr 2020 23:08:02 +0100 Subject: Add support for NFS/EFS mounts (#300) The cc_mounts module does not support NFS mounts in the form of hostname:/ or hostname:/path. This PR adds support for NFS-style paths in the fs_spec field. LP: #1870370 --- cloudinit/config/cc_mounts.py | 13 +++++++++++++ tests/unittests/test_handler/test_handler_mounts.py | 6 ++++++ 2 files changed, 19 insertions(+) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 4ae3f1fc..73537e71 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -74,6 +74,9 @@ from cloudinit import util # Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0 DEVICE_NAME_FILTER = r"^([x]{0,1}[shv]d[a-z][0-9]*|sr[0-9]+)$" DEVICE_NAME_RE = re.compile(DEVICE_NAME_FILTER) +# Name matches 'server:/path' +NETWORK_NAME_FILTER = r"^.+:.*" +NETWORK_NAME_RE = re.compile(NETWORK_NAME_FILTER) WS = re.compile("[%s]+" % (whitespace)) FSTAB_PATH = "/etc/fstab" MNT_COMMENT = "comment=cloudconfig" @@ -93,6 +96,13 @@ def is_meta_device_name(name): return False +def is_network_device(name): + # return true if this is a network device + if NETWORK_NAME_RE.match(name): + return True + return False + + def _get_nth_partition_for_device(device_path, partition_number): potential_suffixes = [str(partition_number), 'p%s' % (partition_number,), '-part%s' % (partition_number,)] @@ -122,6 +132,9 @@ def sanitize_devname(startname, transformer, log): devname = "ephemeral0" log.debug("Adjusted mount option from ephemeral to ephemeral0") + if is_network_device(startname): + return startname + device_path, partition_number = util.expand_dotted_devname(devname) if is_meta_device_name(device_path): diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index 05ac183e..35e72bd1 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -127,6 +127,12 @@ class TestSanitizeDevname(test_helpers.FilesystemMockingTestCase): cc_mounts.sanitize_devname( 'ephemeral0.1', lambda x: disk_path, mock.Mock())) + def test_network_device_returns_network_device(self): + disk_path = 'netdevice:/path' + self.assertEqual( + disk_path, + cc_mounts.sanitize_devname(disk_path, None, mock.Mock())) + class TestFstabHandling(test_helpers.FilesystemMockingTestCase): -- cgit v1.2.3 From 46cf23c28812d3e3ba0c570defd9a05628af5556 Mon Sep 17 00:00:00 2001 From: Eduardo Otubo Date: Tue, 14 Apr 2020 18:16:25 +0200 Subject: swap file "size" being used before checked if str (#315) Swap file size variable was being used before checked if it's set to str "auto". If set to "auto", it will break with: failed to setup swap: unsupported operand type(s) for /: 'str' and 'int' Signed-off-by: Eduardo Otubo RHBZ: 1772505 --- cloudinit/config/cc_mounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 73537e71..1f077bf5 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -287,7 +287,6 @@ def setup_swapfile(fname, size=None, maxsize=None): maxsize: the maximum size """ swap_dir = os.path.dirname(fname) - mibsize = str(int(size / (2 ** 20))) if str(size).lower() == "auto": try: memsize = util.read_meminfo()['total'] @@ -299,6 +298,7 @@ def setup_swapfile(fname, size=None, maxsize=None): size = suggested_swapsize(fsys=swap_dir, maxsize=maxsize, memsize=memsize) + mibsize = str(int(size / (2 ** 20))) if not size: LOG.debug("Not creating swap: suggested size was 0") return -- cgit v1.2.3 From 9d7b35ce23aaf8741dd49b16e359c96591be3c76 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 15 Apr 2020 16:53:08 -0400 Subject: cc_mounts: fix incorrect format specifiers (#316) LP: #1872836 --- cloudinit/config/cc_mounts.py | 8 ++++---- cloudinit/config/tests/test_mounts.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 cloudinit/config/tests/test_mounts.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 1f077bf5..85a89cd1 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -236,20 +236,20 @@ def suggested_swapsize(memsize=None, maxsize=None, fsys=None): return size -def create_swapfile(fname, size): +def create_swapfile(fname: str, size: str) -> None: """Size is in MiB.""" - errmsg = "Failed to create swapfile '%s' of size %dMB via %s: %s" + errmsg = "Failed to create swapfile '%s' of size %sMB via %s: %s" def create_swap(fname, size, method): LOG.debug("Creating swapfile in '%s' on fstype '%s' using '%s'", fname, fstype, method) if method == "fallocate": - cmd = ['fallocate', '-l', '%dM' % size, fname] + cmd = ['fallocate', '-l', '%sM' % size, fname] elif method == "dd": cmd = ['dd', 'if=/dev/zero', 'of=%s' % fname, 'bs=1M', - 'count=%d' % size] + 'count=%s' % size] try: util.subp(cmd, capture=True) diff --git a/cloudinit/config/tests/test_mounts.py b/cloudinit/config/tests/test_mounts.py new file mode 100644 index 00000000..c7dad61a --- /dev/null +++ b/cloudinit/config/tests/test_mounts.py @@ -0,0 +1,22 @@ +# This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + +from cloudinit.config.cc_mounts import create_swapfile + + +M_PATH = 'cloudinit.config.cc_mounts.' + + +class TestCreateSwapfile: + + @mock.patch(M_PATH + 'util.subp') + def test_happy_path(self, m_subp, tmpdir): + swap_file = tmpdir.join("swap-file") + fname = str(swap_file) + + # Some of the calls to util.subp should create the swap file; this + # roughly approximates that + m_subp.side_effect = lambda *args, **kwargs: swap_file.write('') + + create_swapfile(fname, '') + assert mock.call(['mkswap', fname]) in m_subp.call_args_list -- cgit v1.2.3 From 0c5c736727d0f8e363678fe970d70bb889e3893c Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 16 Apr 2020 11:18:36 -0400 Subject: test_mounts: expand happy path test for both happy paths (#319) --- cloudinit/config/tests/test_mounts.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/tests/test_mounts.py b/cloudinit/config/tests/test_mounts.py index c7dad61a..80b54d0f 100644 --- a/cloudinit/config/tests/test_mounts.py +++ b/cloudinit/config/tests/test_mounts.py @@ -1,6 +1,8 @@ # This file is part of cloud-init. See LICENSE file for license information. from unittest import mock +import pytest + from cloudinit.config.cc_mounts import create_swapfile @@ -9,8 +11,10 @@ M_PATH = 'cloudinit.config.cc_mounts.' class TestCreateSwapfile: + @pytest.mark.parametrize('fstype', ('xfs', 'btrfs', 'ext4', 'other')) + @mock.patch(M_PATH + 'util.get_mount_info') @mock.patch(M_PATH + 'util.subp') - def test_happy_path(self, m_subp, tmpdir): + def test_happy_path(self, m_subp, m_get_mount_info, fstype, tmpdir): swap_file = tmpdir.join("swap-file") fname = str(swap_file) @@ -18,5 +22,7 @@ class TestCreateSwapfile: # roughly approximates that m_subp.side_effect = lambda *args, **kwargs: swap_file.write('') + m_get_mount_info.return_value = (mock.ANY, fstype) + create_swapfile(fname, '') assert mock.call(['mkswap', fname]) in m_subp.call_args_list -- cgit v1.2.3 From 38a7e6e9756fdab31264c0d6e93d20432ed111ac Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 24 Apr 2020 09:26:51 -0400 Subject: cloudinit: drop dependencies on unittest2 and contextlib2 (#322) These libraries provide backports of Python 3's stdlib components to Python 2. As we only support Python 3, we can simply use the stdlib now. This pull request does the following: * removes some unneeded compatibility code for the old spelling of `assertRaisesRegex` * replaces invocations of the Python 2-only `assertItemsEqual` with its new name, `assertCountEqual` * replaces all usage of `unittest2` with `unittest` * replaces all usage of `contextlib2` with `contextlib` * drops `unittest2` and `contextlib2` from requirements files and tox.ini It also rewrites some `test_azure` helpers to use bare asserts. We were seeing a strange error in xenial builds of this branch which appear to be stemming from the AssertionError that pytest produces being _different_ from the standard AssertionError. This means that the modified helpers weren't behaving correctly, because they weren't catching AssertionErrors as one would expect. (I believe this is related, in some way, to https://github.com/pytest-dev/pytest/issues/645, but the only version of pytest where we're affected is so far in the past that it's not worth pursuing it any further as we have a workaround.) --- cloudinit/config/tests/test_users_groups.py | 10 ++++---- cloudinit/net/tests/test_dhcp.py | 10 ++++---- cloudinit/net/tests/test_init.py | 2 +- cloudinit/sources/tests/test_init.py | 10 ++++---- cloudinit/tests/helpers.py | 28 ++++++---------------- integration-requirements.txt | 1 - packages/pkg-deps.json | 3 --- test-requirements.txt | 2 -- tests/cloud_tests/testcases/__init__.py | 8 +++---- tests/cloud_tests/testcases/base.py | 12 +++++----- tests/cloud_tests/testcases/modules/ntp_chrony.py | 4 ++-- tests/cloud_tests/verify.py | 4 ++-- tests/unittests/test_builtin_handlers.py | 6 ++--- tests/unittests/test_datasource/test_azure.py | 8 +++---- .../unittests/test_datasource/test_azure_helper.py | 6 ++--- tests/unittests/test_datasource/test_smartos.py | 12 +++++----- .../test_handler/test_handler_ca_certs.py | 6 +---- tests/unittests/test_handler/test_handler_chef.py | 8 +++---- .../test_handler/test_handler_growpart.py | 6 +---- tests/unittests/test_handler/test_schema.py | 4 ++-- tox.ini | 2 -- 21 files changed, 61 insertions(+), 91 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/tests/test_users_groups.py b/cloudinit/config/tests/test_users_groups.py index f620b597..df89ddb3 100644 --- a/cloudinit/config/tests/test_users_groups.py +++ b/cloudinit/config/tests/test_users_groups.py @@ -39,7 +39,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -65,7 +65,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='freebsd', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_fbsd_user.call_args_list, [mock.call('freebsd', groups='wheel', lock_passwd=True, shell='/bin/tcsh'), @@ -86,7 +86,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -107,7 +107,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -146,7 +146,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index c3fa1e04..bc7bef45 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -62,7 +62,7 @@ class TestParseDHCPLeasesFile(CiTestCase): {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) class TestDHCPRFC3442(CiTestCase): @@ -88,7 +88,7 @@ class TestDHCPRFC3442(CiTestCase): 'renew': '4 2017/07/27 18:02:30', 'expire': '5 2017/07/28 07:08:15'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) def test_parse_lease_finds_classless_static_routes(self): """ @@ -114,7 +114,7 @@ class TestDHCPRFC3442(CiTestCase): 'renew': '4 2017/07/27 18:02:30', 'expire': '5 2017/07/28 07:08:15'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') @@ -324,7 +324,7 @@ class TestDHCPDiscoveryClean(CiTestCase): """) write_file(self.tmp_path('dhcp.leases', tmpdir), lease_content) - self.assertItemsEqual( + self.assertCountEqual( [{'interface': 'eth9', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], dhcp_discovery(dhclient_script, 'eth9', tmpdir)) @@ -389,7 +389,7 @@ class TestDHCPDiscoveryClean(CiTestCase): write_file(pid_file, "%d\n" % my_pid) m_getppid.return_value = 1 # Indicate that dhclient has daemonized - self.assertItemsEqual( + self.assertCountEqual( [{'interface': 'eth9', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], dhcp_discovery(dhclient_script, 'eth9', tmpdir)) diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index 32e70b4c..835ed807 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -397,7 +397,7 @@ class TestGetDeviceList(CiTestCase): """get_devicelist returns a directory listing for SYS_CLASS_NET.""" write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), 'up') write_file(os.path.join(self.sysdir, 'eth1', 'operstate'), 'up') - self.assertItemsEqual(['eth0', 'eth1'], net.get_devicelist()) + self.assertCountEqual(['eth0', 'eth1'], net.get_devicelist()) class TestGetInterfaceMAC(CiTestCase): diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py index 5b3648e9..5b6f1b3f 100644 --- a/cloudinit/sources/tests/test_init.py +++ b/cloudinit/sources/tests/test_init.py @@ -350,7 +350,7 @@ class TestDataSource(CiTestCase): 'region': 'myregion', 'some': {'security-credentials': { 'cred1': 'sekret', 'cred2': 'othersekret'}}}) - self.assertItemsEqual( + self.assertCountEqual( ('merged_cfg', 'security-credentials',), datasource.sensitive_metadata_keys) sys_info = { @@ -401,7 +401,7 @@ class TestDataSource(CiTestCase): 'region': 'myregion', 'some': {'security-credentials': REDACT_SENSITIVE_VALUE}}} } - self.assertItemsEqual(expected, redacted) + self.assertCountEqual(expected, redacted) file_stat = os.stat(json_file) self.assertEqual(0o644, stat.S_IMODE(file_stat.st_mode)) @@ -426,7 +426,7 @@ class TestDataSource(CiTestCase): "x86_64"], "variant": "ubuntu", "dist": ["ubuntu", "20.04", "focal"]} - self.assertItemsEqual( + self.assertCountEqual( ('merged_cfg', 'security-credentials',), datasource.sensitive_metadata_keys) with mock.patch("cloudinit.util.system_info", return_value=sys_info): @@ -476,7 +476,7 @@ class TestDataSource(CiTestCase): 'security-credentials': {'cred1': 'sekret', 'cred2': 'othersekret'}}}} } - self.assertItemsEqual(expected, util.load_json(content)) + self.assertCountEqual(expected, util.load_json(content)) file_stat = os.stat(sensitive_json_file) self.assertEqual(0o600, stat.S_IMODE(file_stat.st_mode)) self.assertEqual(expected, util.load_json(content)) @@ -542,7 +542,7 @@ class TestDataSource(CiTestCase): json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp) content = util.load_file(json_file) instance_json = util.load_json(content) - self.assertItemsEqual( + self.assertCountEqual( ['ds/meta_data/key2/key2.1'], instance_json['base64_encoded_keys']) self.assertEqual( diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index f4db5827..477e14c2 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -13,15 +13,10 @@ import string import sys import tempfile import time +import unittest +from contextlib import ExitStack, contextmanager from unittest import mock - -import unittest2 -from unittest2.util import strclass - -try: - from contextlib import ExitStack, contextmanager -except ImportError: - from contextlib2 import ExitStack, contextmanager +from unittest.util import strclass from cloudinit.config.schema import ( SchemaValidationError, validate_cloudconfig_schema) @@ -35,8 +30,8 @@ from cloudinit import util _real_subp = util.subp # Used for skipping tests -SkipTest = unittest2.SkipTest -skipIf = unittest2.skipIf +SkipTest = unittest.SkipTest +skipIf = unittest.skipIf # Makes the old path start @@ -73,7 +68,7 @@ def retarget_many_wrapper(new_base, am, old_func): return wrapper -class TestCase(unittest2.TestCase): +class TestCase(unittest.TestCase): def reset_global_state(self): """Reset any global state to its original settings. @@ -372,7 +367,7 @@ class HttprettyTestCase(CiTestCase): super(HttprettyTestCase, self).tearDown() -class SchemaTestCaseMixin(unittest2.TestCase): +class SchemaTestCaseMixin(unittest.TestCase): def assertSchemaValid(self, cfg, msg="Valid Schema failed validation."): """Assert the config is valid per self.schema. @@ -504,13 +499,4 @@ if not hasattr(mock.Mock, 'assert_not_called'): raise AssertionError(msg) mock.Mock.assert_not_called = __mock_assert_not_called - -# older unittest2.TestCase (centos6) have only the now-deprecated -# assertRaisesRegexp. Simple assignment makes pylint complain, about -# users of assertRaisesRegex so we use getattr to trick it. -# https://github.com/PyCQA/pylint/issues/1946 -if not hasattr(unittest2.TestCase, 'assertRaisesRegex'): - unittest2.TestCase.assertRaisesRegex = ( - getattr(unittest2.TestCase, 'assertRaisesRegexp')) - # vi: ts=4 expandtab diff --git a/integration-requirements.txt b/integration-requirements.txt index 897d6110..44e45c1b 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -5,7 +5,6 @@ # the packages/pkg-deps.json file as well. # -unittest2 # ec2 backend boto3==1.5.9 diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json index d09ec33f..f02e8348 100644 --- a/packages/pkg-deps.json +++ b/packages/pkg-deps.json @@ -10,9 +10,6 @@ "2" : "python-yaml", "3" : "python3-yaml" }, - "contextlib2" : { - "2" : "python-contextlib2" - }, "pyserial" : { "2" : "python-serial", "3" : "python3-serial" diff --git a/test-requirements.txt b/test-requirements.txt index 057115b6..0a6a04d4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,9 +1,7 @@ # Needed generally in tests httpretty>=0.7.1 pytest -unittest2 pytest-cov # Only really needed on older versions of python -contextlib2 setuptools diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py index 6bb39f77..e8c371ca 100644 --- a/tests/cloud_tests/testcases/__init__.py +++ b/tests/cloud_tests/testcases/__init__.py @@ -4,7 +4,7 @@ import importlib import inspect -import unittest2 +import unittest from cloudinit.util import read_conf @@ -48,7 +48,7 @@ def get_test_class(test_name, test_data, test_conf): def __str__(self): return "%s (%s)" % (self._testMethodName, - unittest2.util.strclass(self._realclass)) + unittest.util.strclass(self._realclass)) @classmethod def setUpClass(cls): @@ -62,9 +62,9 @@ def get_suite(test_name, data, conf): @return_value: a test suite """ - suite = unittest2.TestSuite() + suite = unittest.TestSuite() suite.addTest( - unittest2.defaultTestLoader.loadTestsFromTestCase( + unittest.defaultTestLoader.loadTestsFromTestCase( get_test_class(test_name, data, conf))) return suite diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index b0dfc144..7b67f54e 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -5,15 +5,15 @@ import crypt import json import re -import unittest2 +import unittest from cloudinit import util as c_util -SkipTest = unittest2.SkipTest +SkipTest = unittest.SkipTest -class CloudTestCase(unittest2.TestCase): +class CloudTestCase(unittest.TestCase): """Base test class for verifiers.""" # data gets populated in get_suite.setUpClass @@ -172,7 +172,7 @@ class CloudTestCase(unittest2.TestCase): 'Skipping instance-data.json test.' ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) - self.assertItemsEqual(['merged_cfg'], instance_data['sensitive_keys']) + self.assertCountEqual(['merged_cfg'], instance_data['sensitive_keys']) ds = instance_data.get('ds', {}) v1_data = instance_data.get('v1', {}) metadata = ds.get('meta-data', {}) @@ -237,7 +237,7 @@ class CloudTestCase(unittest2.TestCase): ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) v1_data = instance_data.get('v1', {}) - self.assertItemsEqual([], sorted(instance_data['base64_encoded_keys'])) + self.assertCountEqual([], sorted(instance_data['base64_encoded_keys'])) self.assertEqual('unknown', v1_data['cloud_name']) self.assertEqual('lxd', v1_data['platform']) self.assertEqual( @@ -291,7 +291,7 @@ class CloudTestCase(unittest2.TestCase): ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) v1_data = instance_data.get('v1', {}) - self.assertItemsEqual([], instance_data['base64_encoded_keys']) + self.assertCountEqual([], instance_data['base64_encoded_keys']) self.assertEqual('unknown', v1_data['cloud_name']) self.assertEqual('nocloud', v1_data['platform']) subplatform = v1_data['subplatform'] diff --git a/tests/cloud_tests/testcases/modules/ntp_chrony.py b/tests/cloud_tests/testcases/modules/ntp_chrony.py index 0f4c3d08..7d341773 100644 --- a/tests/cloud_tests/testcases/modules/ntp_chrony.py +++ b/tests/cloud_tests/testcases/modules/ntp_chrony.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. """cloud-init Integration Test Verify Script.""" -import unittest2 +import unittest from tests.cloud_tests.testcases import base @@ -13,7 +13,7 @@ class TestNtpChrony(base.CloudTestCase): """Skip this suite of tests on lxd and artful or older.""" if self.platform == 'lxd': if self.is_distro('ubuntu') and self.os_version_cmp('artful') <= 0: - raise unittest2.SkipTest( + raise unittest.SkipTest( 'No support for chrony on containers <= artful.' ' LP: #1589780') return super(TestNtpChrony, self).setUp() diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py index 7018f4d5..0295af40 100644 --- a/tests/cloud_tests/verify.py +++ b/tests/cloud_tests/verify.py @@ -3,7 +3,7 @@ """Verify test results.""" import os -import unittest2 +import unittest from tests.cloud_tests import (config, LOG, util, testcases) @@ -18,7 +18,7 @@ def verify_data(data_dir, platform, os_name, tests): @return_value: {: {passed: True/False, failures: []}} """ base_dir = os.sep.join((data_dir, platform, os_name)) - runner = unittest2.TextTestRunner(verbosity=util.current_verbosity()) + runner = unittest.TextTestRunner(verbosity=util.current_verbosity()) res = {} for test_name in tests: LOG.debug('verifying test data for %s', test_name) diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index b92ffc79..9045e743 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -109,7 +109,7 @@ class TestJinjaTemplatePartHandler(CiTestCase): cloudconfig_handler = CloudConfigPartHandler(self.paths) h = JinjaTemplatePartHandler( self.paths, sub_handlers=[script_handler, cloudconfig_handler]) - self.assertItemsEqual( + self.assertCountEqual( ['text/cloud-config', 'text/cloud-config-jsonp', 'text/x-shellscript'], h.sub_handlers) @@ -120,7 +120,7 @@ class TestJinjaTemplatePartHandler(CiTestCase): cloudconfig_handler = CloudConfigPartHandler(self.paths) h = JinjaTemplatePartHandler( self.paths, sub_handlers=[script_handler, cloudconfig_handler]) - self.assertItemsEqual( + self.assertCountEqual( ['text/cloud-config', 'text/cloud-config-jsonp', 'text/x-shellscript'], h.sub_handlers) @@ -302,7 +302,7 @@ class TestConvertJinjaInstanceData(CiTestCase): expected_data.update({'v1key1': 'v1.1', 'v2key1': 'v2.1'}) converted_data = convert_jinja_instance_data(data=data) - self.assertItemsEqual( + self.assertCountEqual( ['ds', 'v1', 'v2', 'v1key1', 'v2key1'], converted_data.keys()) self.assertEqual( expected_data, diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 2c6aa44d..2f8561cb 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -528,14 +528,14 @@ scbus-1 on xpt0 bus 0 def tags_exists(x, y): for tag in x.keys(): - self.assertIn(tag, y) + assert tag in y for tag in y.keys(): - self.assertIn(tag, x) + assert tag in x def tags_equal(x, y): for x_val in x.values(): y_val = y.get(x_val.tag) - self.assertEqual(x_val.text, y_val.text) + assert x_val.text == y_val.text old_cnt = create_tag_index(oxml) new_cnt = create_tag_index(nxml) @@ -649,7 +649,7 @@ scbus-1 on xpt0 bus 0 crawled_metadata = dsrc.crawl_metadata() - self.assertItemsEqual( + self.assertCountEqual( crawled_metadata.keys(), ['cfg', 'files', 'metadata', 'userdata_raw']) self.assertEqual(crawled_metadata['cfg'], expected_cfg) diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 007df09f..6344b0bb 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import os -import unittest2 +import unittest from textwrap import dedent from cloudinit.sources.helpers import azure as azure_helper @@ -332,7 +332,7 @@ class TestOpenSSLManagerActions(CiTestCase): path = 'tests/data/azure' return os.path.join(path, name) - @unittest2.skip("todo move to cloud_test") + @unittest.skip("todo move to cloud_test") def test_pubkey_extract(self): cert = load_file(self._data_file('pubkey_extract_cert')) good_key = load_file(self._data_file('pubkey_extract_ssh_key')) @@ -344,7 +344,7 @@ class TestOpenSSLManagerActions(CiTestCase): fingerprint = sslmgr._get_fingerprint_from_cert(cert) self.assertEqual(good_fingerprint, fingerprint) - @unittest2.skip("todo move to cloud_test") + @unittest.skip("todo move to cloud_test") @mock.patch.object(azure_helper.OpenSSLManager, '_decrypt_certs_from_xml') def test_parse_certificates(self, mock_decrypt_certs): """Azure control plane puts private keys as well as certificates diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 62084de5..0f0b42bb 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -22,7 +22,7 @@ import os.path import re import signal import stat -import unittest2 +import unittest import uuid from cloudinit import serial @@ -1095,11 +1095,11 @@ class TestNetworkConversion(CiTestCase): self.assertEqual(expected, found) -@unittest2.skipUnless(get_smartos_environ() == SMARTOS_ENV_KVM, - "Only supported on KVM and bhyve guests under SmartOS") -@unittest2.skipUnless(os.access(SERIAL_DEVICE, os.W_OK), - "Requires write access to " + SERIAL_DEVICE) -@unittest2.skipUnless(HAS_PYSERIAL is True, "pyserial not available") +@unittest.skipUnless(get_smartos_environ() == SMARTOS_ENV_KVM, + "Only supported on KVM and bhyve guests under SmartOS") +@unittest.skipUnless(os.access(SERIAL_DEVICE, os.W_OK), + "Requires write access to " + SERIAL_DEVICE) +@unittest.skipUnless(HAS_PYSERIAL is True, "pyserial not available") class TestSerialConcurrency(CiTestCase): """ This class tests locking on an actual serial port, and as such can only diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index 5b4105dd..286ef771 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -11,13 +11,9 @@ import logging import shutil import tempfile import unittest +from contextlib import ExitStack from unittest import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - class TestNoConfig(unittest.TestCase): def setUp(self): diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 2dab3a54..513c18b5 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -65,18 +65,18 @@ class TestInstallChefOmnibus(HttprettyTestCase): cc_chef.install_chef_from_omnibus() expected_kwargs = {'retries': cc_chef.OMNIBUS_URL_RETRIES, 'url': cc_chef.OMNIBUS_URL} - self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[0][1]) + self.assertCountEqual(expected_kwargs, m_rdurl.call_args_list[0][1]) cc_chef.install_chef_from_omnibus(retries=10) expected_kwargs = {'retries': 10, 'url': cc_chef.OMNIBUS_URL} - self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[1][1]) + self.assertCountEqual(expected_kwargs, m_rdurl.call_args_list[1][1]) expected_subp_kwargs = { 'args': ['-v', '2.0'], 'basename': 'chef-omnibus-install', 'blob': m_rdurl.return_value.contents, 'capture': False } - self.assertItemsEqual( + self.assertCountEqual( expected_subp_kwargs, m_subp_blob.call_args_list[0][1]) @@ -97,7 +97,7 @@ class TestInstallChefOmnibus(HttprettyTestCase): 'blob': response, 'capture': False } - self.assertItemsEqual(expected_kwargs, called_kwargs) + self.assertCountEqual(expected_kwargs, called_kwargs) class TestChef(FilesystemMockingTestCase): diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index 43b53745..501bcca5 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -11,13 +11,9 @@ import logging import os import re import unittest +from contextlib import ExitStack from unittest import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - # growpart: # mode: auto # off, on, auto, 'growpart' # devices: ['root'] diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index abde02d3..98628120 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -20,7 +20,7 @@ class GetSchemaTest(CiTestCase): def test_get_schema_coalesces_known_schema(self): """Every cloudconfig module with schema is listed in allOf keyword.""" schema = get_schema() - self.assertItemsEqual( + self.assertCountEqual( [ 'cc_bootcmd', 'cc_ntp', @@ -38,7 +38,7 @@ class GetSchemaTest(CiTestCase): schema['$schema']) # FULL_SCHEMA is updated by the get_schema call from cloudinit.config.schema import FULL_SCHEMA - self.assertItemsEqual(['id', '$schema', 'allOf'], FULL_SCHEMA.keys()) + self.assertCountEqual(['id', '$schema', 'allOf'], FULL_SCHEMA.keys()) def test_get_schema_returns_global_when_set(self): """When FULL_SCHEMA global is already set, get_schema returns it.""" diff --git a/tox.ini b/tox.ini index 416ddcb8..906519c4 100644 --- a/tox.ini +++ b/tox.ini @@ -75,8 +75,6 @@ deps = # test-requirements httpretty==0.9.6 mock==1.3.0 - unittest2==1.1.0 - contextlib2==0.5.1 [testenv:xenial] # When updating this commands definition, also update the definition in -- cgit v1.2.3 From 61a19249c2010743a467a51f8f0750712c2d92e0 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 24 Apr 2020 14:39:25 -0600 Subject: schema: add json schema for write_files module (#152) Add schema definition to cc_write_files.py Cloud-config containing write_files config directives will now emit warnings for invalid config keys or values for the write_files module. Add an extension to JSON schema's draft4validator to permit either binary or text values for 'string' objects. This allows for JSON schema validating the YAML declaration of binary valiues in cloud-config using YAML's '!!binary' syntax. Add the ability to pass a specific module name to `cloud-init devel schema --docs |all` to optionally limit doc output during development to a single schema doc. --- cloudinit/config/cc_write_files.py | 190 +++++++++++++++------ cloudinit/config/schema.py | 106 +++++++++--- tests/unittests/test_cli.py | 4 +- .../test_handler/test_handler_write_files.py | 85 ++++++++- tests/unittests/test_handler/test_schema.py | 7 +- 5 files changed, 314 insertions(+), 78 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index bd87e9e5..16ea80e2 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -4,60 +4,14 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Write Files ------------ -**Summary:** write arbitrary files - -Write out arbitrary content to files, optionally setting permissions. Content -can be specified in plain text or binary. Data encoded with either base64 or -binary gzip data can be specified and will be decoded before being written. - -.. note:: - if multiline data is provided, care should be taken to ensure that it - follows yaml formatting standards. to specify binary data, use the yaml - option ``!!binary`` - -.. note:: - Do not write files under /tmp during boot because of a race with - systemd-tmpfiles-clean that can cause temp files to get cleaned during - the early boot process. Use /run/somedir instead to avoid race LP:1707222. - -**Internal name:** ``cc_write_files`` - -**Module frequency:** per instance - -**Supported distros:** all - -**Config keys**:: - - write_files: - - encoding: b64 - content: CiMgVGhpcyBmaWxlIGNvbnRyb2xzIHRoZSBzdGF0ZSBvZiBTRUxpbnV4... - owner: root:root - path: /etc/sysconfig/selinux - permissions: '0644' - - content: | - # My new /etc/sysconfig/samba file - - SMDBOPTIONS="-D" - path: /etc/sysconfig/samba - - content: !!binary | - f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAwARAAAAAAABAAAAAAAAAAJAVAAAAAA - AEAAHgAdAAYAAAAFAAAAQAAAAAAAAABAAEAAAAAAAEAAQAAAAAAAwAEAAAAAAA - AAAAAAAAAwAAAAQAAAAAAgAAAAAAAAACQAAAAAAAAAJAAAAAAAAcAAAAAAAAAB - ... - path: /bin/arch - permissions: '0555' - - content: | - 15 * * * * root ship_logs - path: /etc/crontab - append: true -""" +"""Write Files: write arbitrary files""" import base64 import os +from textwrap import dedent +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE from cloudinit import util @@ -71,6 +25,141 @@ UNKNOWN_ENC = 'text/plain' LOG = logging.getLogger(__name__) +distros = ['all'] + +# The schema definition for each cloud-config module is a strict contract for +# describing supported configuration parameters for each cloud-config section. +# It allows cloud-config to validate and alert users to invalid or ignored +# configuration options before actually attempting to deploy with said +# configuration. + +supported_encoding_types = [ + 'gz', 'gzip', 'gz+base64', 'gzip+base64', 'gz+b64', 'gzip+b64', 'b64', + 'base64'] + +schema = { + 'id': 'cc_write_files', + 'name': 'Write Files', + 'title': 'write arbitrary files', + 'description': dedent("""\ + Write out arbitrary content to files, optionally setting permissions. + Content can be specified in plain text or binary. Data encoded with + either base64 or binary gzip data can be specified and will be decoded + before being written. For empty file creation, content can be omitted. + + .. note:: + if multiline data is provided, care should be taken to ensure that it + follows yaml formatting standards. to specify binary data, use the yaml + option ``!!binary`` + + .. note:: + Do not write files under /tmp during boot because of a race with + systemd-tmpfiles-clean that can cause temp files to get cleaned during + the early boot process. Use /run/somedir instead to avoid race + LP:1707222."""), + 'distros': distros, + 'examples': [ + dedent("""\ + # Write out base64 encoded content to /etc/sysconfig/selinux + write_files: + - encoding: b64 + content: CiMgVGhpcyBmaWxlIGNvbnRyb2xzIHRoZSBzdGF0ZSBvZiBTRUxpbnV4... + owner: root:root + path: /etc/sysconfig/selinux + permissions: '0644' + """), + dedent("""\ + # Appending content to an existing file + write_files: + - content: | + 15 * * * * root ship_logs + path: /etc/crontab + append: true + """), + dedent("""\ + # Provide gziped binary content + write_files: + - encoding: gzip + content: !!binary | + H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA= + path: /usr/bin/hello + permissions: '0755' + """), + dedent("""\ + # Create an empty file on the system + write_files: + - path: /root/CLOUD_INIT_WAS_HERE + """)], + 'frequency': frequency, + 'type': 'object', + 'properties': { + 'write_files': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'path': { + 'type': 'string', + 'description': dedent("""\ + Path of the file to which ``content`` is decoded + and written + """), + }, + 'content': { + 'type': 'string', + 'default': '', + 'description': dedent("""\ + Optional content to write to the provided ``path``. + When content is present and encoding is not '%s', + decode the content prior to writing. Default: + **''** + """ % UNKNOWN_ENC), + }, + 'owner': { + 'type': 'string', + 'default': DEFAULT_OWNER, + 'description': dedent("""\ + Optional owner:group to chown on the file. Default: + **{owner}** + """.format(owner=DEFAULT_OWNER)), + }, + 'permissions': { + 'type': 'string', + 'default': oct(DEFAULT_PERMS).replace('o', ''), + 'description': dedent("""\ + Optional file permissions to set on ``path`` + represented as an octal string '0###'. Default: + **'{perms}'** + """.format(perms=oct(DEFAULT_PERMS).replace('o', ''))), + }, + 'encoding': { + 'type': 'string', + 'default': UNKNOWN_ENC, + 'enum': supported_encoding_types, + 'description': dedent("""\ + Optional encoding type of the content. Default is + **text/plain** and no content decoding is + performed. Supported encoding types are: + %s.""" % ", ".join(supported_encoding_types)), + }, + 'append': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + Whether to append ``content`` to existing file if + ``path`` exists. Default: **false**. + """), + }, + }, + 'required': ['path'], + 'additionalProperties': False + }, + } + } +} + +__doc__ = get_schema_doc(schema) # Supplement python help() + def handle(name, cfg, _cloud, log, _args): files = cfg.get('write_files') @@ -78,6 +167,7 @@ def handle(name, cfg, _cloud, log, _args): log.debug(("Skipping module named %s," " no/empty 'write_files' key in configuration"), name) return + validate_cloudconfig_schema(cfg, schema) write_files(name, files) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 807c3eee..5356f492 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -36,6 +36,8 @@ SCHEMA_DOC_TMPL = """ {examples} """ SCHEMA_PROPERTY_TMPL = '{prefix}**{prop_name}:** ({type}) {description}' +SCHEMA_LIST_ITEM_TMPL = ( + '{prefix}Each item in **{prop_name}** list supports the following keys:') SCHEMA_EXAMPLES_HEADER = '\n**Examples**::\n\n' SCHEMA_EXAMPLES_SPACER_TEMPLATE = '\n # --- Example{0} ---' @@ -58,6 +60,19 @@ class SchemaValidationError(ValueError): super(SchemaValidationError, self).__init__(message) +def is_schema_byte_string(checker, instance): + """TYPE_CHECKER override allowing bytes for string type + + For jsonschema v. 3.0.0+ + """ + try: + from jsonschema import Draft4Validator + except ImportError: + return False + return (Draft4Validator.TYPE_CHECKER.is_type(instance, "string") or + isinstance(instance, (bytes,))) + + def validate_cloudconfig_schema(config, schema, strict=False): """Validate provided config meets the schema definition. @@ -73,11 +88,28 @@ def validate_cloudconfig_schema(config, schema, strict=False): """ try: from jsonschema import Draft4Validator, FormatChecker + from jsonschema.validators import create, extend except ImportError: logging.debug( 'Ignoring schema validation. python-jsonschema is not present') return - validator = Draft4Validator(schema, format_checker=FormatChecker()) + + # Allow for bytes to be presented as an acceptable valid value for string + # type jsonschema attributes in cloud-init's schema. + # This allows #cloud-config to provide valid yaml "content: !!binary | ..." + if hasattr(Draft4Validator, 'TYPE_CHECKER'): # jsonschema 3.0+ + type_checker = Draft4Validator.TYPE_CHECKER.redefine( + 'string', is_schema_byte_string) + cloudinitValidator = extend(Draft4Validator, type_checker=type_checker) + else: # jsonschema 2.6 workaround + types = Draft4Validator.DEFAULT_TYPES + types['string'] = (str, bytes) # Allow bytes as well as string + cloudinitValidator = create( + meta_schema=Draft4Validator.META_SCHEMA, + validators=Draft4Validator.VALIDATORS, + version="draft4", + default_types=types) + validator = cloudinitValidator(schema, format_checker=FormatChecker()) errors = () for error in sorted(validator.iter_errors(config), key=lambda e: e.path): path = '.'.join([str(p) for p in error.path]) @@ -106,7 +138,6 @@ def annotated_cloudconfig_file(cloudconfig, original_content, schema_errors): schemapaths = _schemapath_for_cloudconfig( cloudconfig, original_content) errors_by_line = defaultdict(list) - error_count = 1 error_footer = [] annotated_content = [] for path, msg in schema_errors: @@ -120,18 +151,17 @@ def annotated_cloudconfig_file(cloudconfig, original_content, schema_errors): if col is not None: msg = 'Line {line} column {col}: {msg}'.format( line=line, col=col, msg=msg) - error_footer.append('# E{0}: {1}'.format(error_count, msg)) - error_count += 1 lines = original_content.decode().split('\n') - error_count = 1 - for line_number, line in enumerate(lines): - errors = errors_by_line[line_number + 1] + error_index = 1 + for line_number, line in enumerate(lines, 1): + errors = errors_by_line[line_number] if errors: - error_label = ','.join( - ['E{0}'.format(count + error_count) - for count in range(0, len(errors))]) - error_count += len(errors) - annotated_content.append(line + '\t\t# ' + error_label) + error_label = [] + for error in errors: + error_label.append('E{0}'.format(error_index)) + error_footer.append('# E{0}: {1}'.format(error_index, error)) + error_index += 1 + annotated_content.append(line + '\t\t# ' + ','.join(error_label)) else: annotated_content.append(line) annotated_content.append( @@ -213,20 +243,34 @@ def _schemapath_for_cloudconfig(config, original_content): previous_depth = -1 path_prefix = '' if line.startswith('- '): + # Process list items adding a list_index to the path prefix + previous_list_idx = '.%d' % (list_index - 1) + if path_prefix and path_prefix.endswith(previous_list_idx): + path_prefix = path_prefix[:-len(previous_list_idx)] key = str(list_index) - value = line[1:] + schema_line_numbers[key] = line_number + item_indent = len(re.match(RE_YAML_INDENT, line[1:]).groups()[0]) + item_indent += 1 # For the leading '-' character + previous_depth = indent_depth + indent_depth += item_indent + line = line[item_indent:] # Strip leading list item + whitespace list_index += 1 else: + # Process non-list lines setting value if present list_index = 0 key, value = line.split(':', 1) + if path_prefix: + # Append any existing path_prefix for a fully-pathed key + key = path_prefix + '.' + key while indent_depth <= previous_depth: if scopes: previous_depth, path_prefix = scopes.pop() + if list_index > 0 and indent_depth == previous_depth: + path_prefix = '.'.join(path_prefix.split('.')[:-1]) + break else: previous_depth = -1 path_prefix = '' - if path_prefix: - key = path_prefix + '.' + key scopes.append((indent_depth, key)) if value: value = value.strip() @@ -266,11 +310,23 @@ def _get_property_doc(schema, prefix=' '): for prop_key, prop_config in schema.get('properties', {}).items(): # Define prop_name and dscription for SCHEMA_PROPERTY_TMPL description = prop_config.get('description', '') + properties.append(SCHEMA_PROPERTY_TMPL.format( prefix=prefix, prop_name=prop_key, type=_get_property_type(prop_config), description=description.replace('\n', ''))) + items = prop_config.get('items') + if items: + if isinstance(items, list): + for item in items: + properties.append( + _get_property_doc(item, prefix=new_prefix)) + elif isinstance(items, dict) and items.get('properties'): + properties.append(SCHEMA_LIST_ITEM_TMPL.format( + prefix=new_prefix, prop_name=prop_key)) + new_prefix += ' ' + properties.append(_get_property_doc(items, prefix=new_prefix)) if 'properties' in prop_config: properties.append( _get_property_doc(prop_config, prefix=new_prefix)) @@ -346,8 +402,9 @@ def get_parser(parser=None): description='Validate cloud-config files or document schema') parser.add_argument('-c', '--config-file', help='Path of the cloud-config yaml file to validate') - parser.add_argument('-d', '--doc', action="store_true", default=False, - help='Print schema documentation') + parser.add_argument('-d', '--docs', nargs='+', + help=('Print schema module docs. Choices: all or' + ' space-delimited cc_names.')) parser.add_argument('--annotate', action="store_true", default=False, help='Annotate existing cloud-config file with errors') return parser @@ -355,9 +412,9 @@ def get_parser(parser=None): def handle_schema_args(name, args): """Handle provided schema args and perform the appropriate actions.""" - exclusive_args = [args.config_file, args.doc] + exclusive_args = [args.config_file, args.docs] if not any(exclusive_args) or all(exclusive_args): - error('Expected either --config-file argument or --doc') + error('Expected either --config-file argument or --docs') full_schema = get_schema() if args.config_file: try: @@ -370,9 +427,16 @@ def handle_schema_args(name, args): error(str(e)) else: print("Valid cloud-config file {0}".format(args.config_file)) - if args.doc: + elif args.docs: + schema_ids = [subschema['id'] for subschema in full_schema['allOf']] + schema_ids += ['all'] + invalid_docs = set(args.docs).difference(set(schema_ids)) + if invalid_docs: + error('Invalid --docs value {0}. Must be one of: {1}'.format( + list(invalid_docs), ', '.join(schema_ids))) for subschema in full_schema['allOf']: - print(get_schema_doc(subschema)) + if 'all' in args.docs or subschema['id'] in args.docs: + print(get_schema_doc(subschema)) def main(): diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index e57c15d1..43d996b9 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -214,14 +214,14 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): self.assertEqual(1, exit_code) # Known whitebox output from schema subcommand self.assertEqual( - 'Expected either --config-file argument or --doc\n', + 'Expected either --config-file argument or --docs\n', self.stderr.getvalue()) def test_wb_devel_schema_subcommand_doc_content(self): """Validate that doc content is sane from known examples.""" stdout = io.StringIO() self.patchStdoutAndStderr(stdout=stdout) - self._call_main(['cloud-init', 'devel', 'schema', '--doc']) + self._call_main(['cloud-init', 'devel', 'schema', '--docs', 'all']) expected_doc_sections = [ '**Supported distros:** all', '**Supported distros:** centos, debian, fedora', diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py index ed0a4da2..727681d3 100644 --- a/tests/unittests/test_handler/test_handler_write_files.py +++ b/tests/unittests/test_handler/test_handler_write_files.py @@ -1,15 +1,19 @@ # This file is part of cloud-init. See LICENSE file for license information. import base64 +import copy import gzip import io import shutil import tempfile +from cloudinit.config.cc_write_files import ( + handle, decode_perms, write_files) from cloudinit import log as logging from cloudinit import util -from cloudinit.config.cc_write_files import write_files, decode_perms -from cloudinit.tests.helpers import CiTestCase, FilesystemMockingTestCase + +from cloudinit.tests.helpers import ( + CiTestCase, FilesystemMockingTestCase, mock, skipUnlessJsonSchema) LOG = logging.getLogger(__name__) @@ -36,13 +40,90 @@ YAML_CONTENT_EXPECTED = { '/tmp/message': "hi mom line 1\nhi mom line 2\n", } +VALID_SCHEMA = { + 'write_files': [ + {'append': False, 'content': 'a', 'encoding': 'gzip', 'owner': 'jeff', + 'path': '/some', 'permissions': '0777'} + ] +} + +INVALID_SCHEMA = { # Dropped required path key + 'write_files': [ + {'append': False, 'content': 'a', 'encoding': 'gzip', 'owner': 'jeff', + 'permissions': '0777'} + ] +} + + +@skipUnlessJsonSchema() +@mock.patch('cloudinit.config.cc_write_files.write_files') +class TestWriteFilesSchema(CiTestCase): + + with_logs = True + + def test_schema_validation_warns_missing_path(self, m_write_files): + """The only required file item property is 'path'.""" + cc = self.tmp_cloud('ubuntu') + valid_config = {'write_files': [{'path': '/some/path'}]} + handle('cc_write_file', valid_config, cc, LOG, []) + self.assertNotIn('Invalid config:', self.logs.getvalue()) + handle('cc_write_file', INVALID_SCHEMA, cc, LOG, []) + self.assertIn('Invalid config:', self.logs.getvalue()) + self.assertIn("'path' is a required property", self.logs.getvalue()) + + def test_schema_validation_warns_non_string_type_for_files( + self, m_write_files): + """Schema validation warns of non-string values for each file item.""" + cc = self.tmp_cloud('ubuntu') + for key in VALID_SCHEMA['write_files'][0].keys(): + if key == 'append': + key_type = 'boolean' + else: + key_type = 'string' + invalid_config = copy.deepcopy(VALID_SCHEMA) + invalid_config['write_files'][0][key] = 1 + handle('cc_write_file', invalid_config, cc, LOG, []) + self.assertIn( + mock.call('cc_write_file', invalid_config['write_files']), + m_write_files.call_args_list) + self.assertIn( + 'write_files.0.%s: 1 is not of type \'%s\'' % (key, key_type), + self.logs.getvalue()) + self.assertIn('Invalid config:', self.logs.getvalue()) + + def test_schema_validation_warns_on_additional_undefined_propertes( + self, m_write_files): + """Schema validation warns on additional undefined file properties.""" + cc = self.tmp_cloud('ubuntu') + invalid_config = copy.deepcopy(VALID_SCHEMA) + invalid_config['write_files'][0]['bogus'] = 'value' + handle('cc_write_file', invalid_config, cc, LOG, []) + self.assertIn( + "Invalid config:\nwrite_files.0: Additional properties" + " are not allowed ('bogus' was unexpected)", + self.logs.getvalue()) + class TestWriteFiles(FilesystemMockingTestCase): + + with_logs = True + def setUp(self): super(TestWriteFiles, self).setUp() self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) + @skipUnlessJsonSchema() + def test_handler_schema_validation_warns_non_array_type(self): + """Schema validation warns of non-array value.""" + invalid_config = {'write_files': 1} + cc = self.tmp_cloud('ubuntu') + with self.assertRaises(TypeError): + handle('cc_write_file', invalid_config, cc, LOG, []) + self.assertIn( + 'Invalid config:\nwrite_files: 1 is not of type \'array\'', + self.logs.getvalue()) + def test_simple(self): self.patchUtils(self.tmp) expected = "hello world\n" diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 98628120..2d4536d9 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -29,6 +29,7 @@ class GetSchemaTest(CiTestCase): 'cc_snap', 'cc_ubuntu_advantage', 'cc_ubuntu_drivers', + 'cc_write_files', 'cc_zypper_add_repo' ], [subschema['id'] for subschema in schema['allOf']]) @@ -351,7 +352,7 @@ class MainTest(CiTestCase): main() self.assertEqual(1, context_manager.exception.code) self.assertEqual( - 'Expected either --config-file argument or --doc\n', + 'Expected either --config-file argument or --docs\n', m_stderr.getvalue()) def test_main_absent_config_file(self): @@ -367,8 +368,8 @@ class MainTest(CiTestCase): m_stderr.getvalue()) def test_main_prints_docs(self): - """When --doc parameter is provided, main generates documentation.""" - myargs = ['mycmd', '--doc'] + """When --docs parameter is provided, main generates documentation.""" + myargs = ['mycmd', '--docs', 'all'] with mock.patch('sys.argv', myargs): with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: self.assertEqual(0, main(), 'Expected 0 exit code') -- cgit v1.2.3 From 429c1ab9b0ee04f5a2287b8579c02907f30e795f Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 27 Apr 2020 16:01:10 -0400 Subject: schema: ignore spurious pylint error (#332) The line in question is in the code path handling older versions of jsonschema. In that context it _is_ correct, but when pylint analyses it against the latest jsonschema it (incorrectly) detects an error. --- cloudinit/config/schema.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 5356f492..a295d63d 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -103,7 +103,10 @@ def validate_cloudconfig_schema(config, schema, strict=False): cloudinitValidator = extend(Draft4Validator, type_checker=type_checker) else: # jsonschema 2.6 workaround types = Draft4Validator.DEFAULT_TYPES - types['string'] = (str, bytes) # Allow bytes as well as string + # Allow bytes as well as string (and disable a spurious + # unsupported-assignment-operation pylint warning which appears because + # this code path isn't written against the latest jsonschema). + types['string'] = (str, bytes) # pylint: disable=E1137 cloudinitValidator = create( meta_schema=Draft4Validator.META_SCHEMA, validators=Draft4Validator.VALIDATORS, -- cgit v1.2.3 From b70f11079e6d4264ff9a8c12089945f57d903e28 Mon Sep 17 00:00:00 2001 From: Adrian Wilkins Date: Tue, 28 Apr 2020 16:55:34 +0100 Subject: Add docs about creating parent folders (#330) Clarify in documentation that write_files will create parent folders for paths that do not already exist. This obfuscates what the problem is when people erroneously create files in /tmp despite the warnings in the documentation not to do so. People naturally assume that their file is absent because the parent folder did not exist for it to be created in, causing them to add a runcmd block to create the folder, even though execution order means that this will not occur until after write_files have all finished. --- cloudinit/config/cc_write_files.py | 1 + 1 file changed, 1 insertion(+) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 16ea80e2..204cbfd6 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -43,6 +43,7 @@ schema = { 'title': 'write arbitrary files', 'description': dedent("""\ Write out arbitrary content to files, optionally setting permissions. + Parent folders in the path are created if absent. Content can be specified in plain text or binary. Data encoded with either base64 or binary gzip data can be specified and will be decoded before being written. For empty file creation, content can be omitted. -- cgit v1.2.3 From 5815dd9bb859859573d9d3219918e5ab1ce3d3af Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 30 Apr 2020 09:24:42 -0500 Subject: yum_add_repo: Add Centos to the supported distro list (#340) Users of Centos who want to add yum repos, like they do on Fedora or RHEL get this unfortunate message: Skipping modules 'yum-add-repo' because they are not verified on distro 'centos'. To run anyway, add them to 'unverified_modules' in config Centos certainly supports yum, add it to the supported distro list in the module. --- cloudinit/config/cc_yum_add_repo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 3673166a..01fe683c 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -18,7 +18,7 @@ entry, the config entry will be skipped. **Module frequency:** per always -**Supported distros:** fedora, rhel +**Supported distros:** centos, fedora, rhel **Config keys**:: @@ -36,7 +36,7 @@ from configparser import ConfigParser from cloudinit import util -distros = ['fedora', 'rhel'] +distros = ['centos', 'fedora', 'rhel'] def _canonicalize_id(repo_id): -- cgit v1.2.3 From 7dceb9882590fb738ac0ff3429908cc6c945485a Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 30 Apr 2020 17:04:48 -0400 Subject: cc_locale: introduce schema (#335) --- cloudinit/config/cc_locale.py | 65 ++++++++++++++++++++++------- tests/unittests/test_handler/test_schema.py | 1 + 2 files changed, 50 insertions(+), 16 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_locale.py b/cloudinit/config/cc_locale.py index f68c3cc7..4f8b7bf6 100644 --- a/cloudinit/config/cc_locale.py +++ b/cloudinit/config/cc_locale.py @@ -6,27 +6,58 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Locale ------- -**Summary:** set system locale +"""Locale: set system locale""" -Configure the system locale and apply it system wide. By default use the locale -specified by the datasource. +from textwrap import dedent -**Internal name:** ``cc_locale`` - -**Module frequency:** per instance +from cloudinit import util +from cloudinit.config.schema import get_schema_doc, validate_cloudconfig_schema +from cloudinit.settings import PER_INSTANCE -**Supported distros:** all -**Config keys**:: +frequency = PER_INSTANCE +distros = ['all'] +schema = { + 'id': 'cc_locale', + 'name': 'Locale', + 'title': 'Set system locale', + 'description': dedent( + """\ + Configure the system locale and apply it system wide. By default use + the locale specified by the datasource.""" + ), + 'distros': distros, + 'examples': [ + dedent("""\ + # Set the locale to ar_AE + locale: ar_AE + """), + dedent("""\ + # Set the locale to fr_CA in /etc/alternate_path/locale + locale: fr_CA + locale_configfile: /etc/alternate_path/locale + """), + ], + 'frequency': frequency, + 'type': 'object', + 'properties': { + 'locale': { + 'type': 'string', + 'description': ( + "The locale to set as the system's locale (e.g. ar_PS)" + ), + }, + 'locale_configfile': { + 'type': 'string', + 'description': ( + "The file in which to write the locale configuration (defaults" + " to the distro's default location)" + ), + }, + }, +} - locale: - locale_configfile: -""" - -from cloudinit import util +__doc__ = get_schema_doc(schema) # Supplement python help() def handle(name, cfg, cloud, log, args): @@ -40,6 +71,8 @@ def handle(name, cfg, cloud, log, args): name, locale) return + validate_cloudconfig_schema(cfg, schema) + log.debug("Setting locale to %s", locale) locale_cfgfile = util.get_cfg_option_str(cfg, "locale_configfile") cloud.distro.apply_locale(locale, locale_cfgfile) diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 2d4536d9..1059fb8b 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -23,6 +23,7 @@ class GetSchemaTest(CiTestCase): self.assertCountEqual( [ 'cc_bootcmd', + 'cc_locale', 'cc_ntp', 'cc_resizefs', 'cc_runcmd', -- cgit v1.2.3 From 022122a54e895468d054ed162fdf1b403b7aaf3f Mon Sep 17 00:00:00 2001 From: lucasmoura Date: Wed, 6 May 2020 17:19:35 -0300 Subject: Create tests to validate schema examples (#348) Add a unit test to validate if the examples provided in the config modules are conforming to the concatenated schema of all config modules. The rationale behind that is not only to verify if the examples are correctly written but to assert that no config schema is interfering with each other. Failures in validate_cloudconfig_schema raise the SchemaValidationError by using strict=True, so I have only called the function passing the right schema examples to validate. This branch also fixes an invalid schema example in cc_snap. LP: #1876412 --- cloudinit/config/cc_snap.py | 4 ++-- tests/unittests/test_handler/test_schema.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py index 90724b81..fbe7787a 100644 --- a/cloudinit/config/cc_snap.py +++ b/cloudinit/config/cc_snap.py @@ -61,9 +61,9 @@ schema = { snap: assertions: 00: | - signed_assertion_blob_here + signed_assertion_blob_here 02: | - signed_assertion_blob_here + signed_assertion_blob_here commands: 00: snap create-user --sudoer --known @mydomain.com 01: snap install canonical-livepatch diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 1059fb8b..ae3838d8 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -10,6 +10,7 @@ from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema from copy import copy import os +import pytest from io import StringIO from textwrap import dedent from yaml import safe_load @@ -112,6 +113,24 @@ class ValidateCloudConfigSchemaTest(CiTestCase): str(context_mgr.exception)) +class TestCloudConfigExamples: + + SCHEMA = get_schema() + PARAMS = [ + (schema["id"], example) + for schema in SCHEMA["allOf"] for example in schema["examples"]] + + @pytest.mark.parametrize("schema_id,example", PARAMS) + @skipUnlessJsonSchema() + def test_validateconfig_schema_of_example(self, schema_id, example): + """ For a given example in a config module we test if it is valid + according to the unified schema of all config modules + """ + config_load = safe_load(example) + validate_cloudconfig_schema( + config_load, TestCloudConfigExamples.SCHEMA, strict=True) + + class ValidateCloudConfigFileTest(CiTestCase): """Tests for validate_cloudconfig_file.""" -- cgit v1.2.3 From 2e32c40a607250bc9e713c0daf360dc6617f4420 Mon Sep 17 00:00:00 2001 From: lucasmoura Date: Wed, 13 May 2020 17:45:01 -0300 Subject: Add schema to apt configure config (#357) Create a schema object for the `apt_configure` module and validate this schema in the `handle` function of the module. There are some considerations regarding this PR: * The `primary` and `security` keys have the exact same properties. I tried to eliminate this redundancy by moving their properties to a common place and then just referencing it for both security and primary. Similar to what is documented here: https://json-schema.org/understanding-json-schema/structuring.html under the `Reuse` paragraph. However, this approach does not work, because the `#` pointer goes to the beginning of the file, which is a python module instead of a json file, not allowing the pointer to find the correct definition. What I did was to create a separate dict for the mirror config and reuse it for primary and security, but maybe there are better approaches to do that. * There was no documentation for the config `debconf_selections`. I tried to infer what it supposed to do by looking at the code and the `debconf-set-selections` manpage, but my description may not be accurate or complete. * Add a _parse_description function to schema.py to render multi-line preformatted content instead of squashing all whitespace LP: #1858884 --- cloudinit/config/cc_apt_configure.py | 538 +++++++++++++-------- cloudinit/config/cc_ntp.py | 42 +- cloudinit/config/cc_write_files.py | 22 +- cloudinit/config/schema.py | 24 +- doc/examples/cloud-config-apt.txt | 2 +- doc/examples/cloud-config-chef-oneiric.txt | 63 +-- doc/examples/cloud-config.txt | 9 +- .../examples/install_run_chef_recipes.yaml | 73 +-- tests/unittests/test_handler/test_schema.py | 36 ++ 9 files changed, 507 insertions(+), 302 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ff58c062..9a33451d 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -6,228 +6,371 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Apt Configure -------------- -**Summary:** configure apt - -This module handles both configuration of apt options and adding source lists. -There are configuration options such as ``apt_get_wrapper`` and -``apt_get_command`` that control how cloud-init invokes apt-get. -These configuration options are handled on a per-distro basis, so consult -documentation for cloud-init's distro support for instructions on using -these config options. - -.. note:: - To ensure that apt configuration is valid yaml, any strings containing - special characters, especially ``:`` should be quoted. - -.. note:: - For more information about apt configuration, see the - ``Additional apt configuration`` example. - -**Preserve sources.list:** - -By default, cloud-init will generate a new sources list in -``/etc/apt/sources.list.d`` based on any changes specified in cloud config. -To disable this behavior and preserve the sources list from the pristine image, -set ``preserve_sources_list`` to ``true``. - -.. note:: - The ``preserve_sources_list`` option overrides all other config keys that - would alter ``sources.list`` or ``sources.list.d``, **except** for - additional sources to be added to ``sources.list.d``. - -**Disable source suites:** - -Entries in the sources list can be disabled using ``disable_suites``, which -takes a list of suites to be disabled. If the string ``$RELEASE`` is present in -a suite in the ``disable_suites`` list, it will be replaced with the release -name. If a suite specified in ``disable_suites`` is not present in -``sources.list`` it will be ignored. For convenience, several aliases are -provided for ``disable_suites``: - - - ``updates`` => ``$RELEASE-updates`` - - ``backports`` => ``$RELEASE-backports`` - - ``security`` => ``$RELEASE-security`` - - ``proposed`` => ``$RELEASE-proposed`` - - ``release`` => ``$RELEASE`` - -.. note:: - When a suite is disabled using ``disable_suites``, its entry in - ``sources.list`` is not deleted; it is just commented out. - -**Configure primary and security mirrors:** - -The primary and security archive mirrors can be specified using the ``primary`` -and ``security`` keys, respectively. Both the ``primary`` and ``security`` keys -take a list of configs, allowing mirrors to be specified on a per-architecture -basis. Each config is a dictionary which must have an entry for ``arches``, -specifying which architectures that config entry is for. The keyword -``default`` applies to any architecture not explicitly listed. The mirror url -can be specified with the ``uri`` key, or a list of mirrors to check can be -provided in order, with the first mirror that can be resolved being selected. -This allows the same configuration to be used in different environment, with -different hosts used for a local apt mirror. If no mirror is provided by -``uri`` or ``search``, ``search_dns`` may be used to search for dns names in -the format ``-mirror`` in each of the following: - - - fqdn of this host per cloud metadata - - localdomain - - domains listed in ``/etc/resolv.conf`` - -If there is a dns entry for ``-mirror``, then it is assumed that there -is a distro mirror at ``http://-mirror./``. If the -``primary`` key is defined, but not the ``security`` key, then then -configuration for ``primary`` is also used for ``security``. If ``search_dns`` -is used for the ``security`` key, the search pattern will be. -``-security-mirror``. - -If no mirrors are specified, or all lookups fail, then default mirrors defined -in the datasource are used. If none are present in the datasource either the -following defaults are used: - - - primary: ``http://archive.ubuntu.com/ubuntu`` - - security: ``http://security.ubuntu.com/ubuntu`` - -**Specify sources.list template:** - -A custom template for rendering ``sources.list`` can be specefied with -``sources_list``. If no ``sources_list`` template is given, cloud-init will -use sane default. Within this template, the following strings will be replaced -with the appropriate values: - - - ``$MIRROR`` - - ``$RELEASE`` - - ``$PRIMARY`` - - ``$SECURITY`` - -**Pass configuration to apt:** - -Apt configuration can be specified using ``conf``. Configuration is specified -as a string. For multiline apt configuration, make sure to follow yaml syntax. - -**Configure apt proxy:** - -Proxy configuration for apt can be specified using ``conf``, but proxy config -keys also exist for convenience. The proxy config keys, ``http_proxy``, -``ftp_proxy``, and ``https_proxy`` may be used to specify a proxy for http, ftp -and https protocols respectively. The ``proxy`` key also exists as an alias for -``http_proxy``. Proxy url is specified in the format -``://[[user][:pass]@]host[:port]/``. - -**Add apt repos by regex:** +"""Apt Configure: Configure apt for the user.""" -All source entries in ``apt-sources`` that match regex in -``add_apt_repo_match`` will be added to the system using -``add-apt-repository``. If ``add_apt_repo_match`` is not specified, it defaults -to ``^[\\w-]+:\\w`` - -**Add source list entries:** - -Source list entries can be specified as a dictionary under the ``sources`` -config key, with key in the dict representing a different source file. The key -of each source entry will be used as an id that can be referenced in -other config entries, as well as the filename for the source's configuration -under ``/etc/apt/sources.list.d``. If the name does not end with ``.list``, -it will be appended. If there is no configuration for a key in ``sources``, no -file will be written, but the key may still be referred to as an id in other -``sources`` entries. - -Each entry under ``sources`` is a dictionary which may contain any of the -following optional keys: - - - ``source``: a sources.list entry (some variable replacements apply) - - ``keyid``: a key to import via shortid or fingerprint - - ``key``: a raw PGP key - - ``keyserver``: alternate keyserver to pull ``keyid`` key from - -The ``source`` key supports variable replacements for the following strings: - - - ``$MIRROR`` - - ``$PRIMARY`` - - ``$SECURITY`` - - ``$RELEASE`` - -**Internal name:** ``cc_apt_configure`` +import glob +import os +import re +from textwrap import dedent -**Module frequency:** per instance +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) +from cloudinit import gpg +from cloudinit import log as logging +from cloudinit import templater +from cloudinit import util +from cloudinit.settings import PER_INSTANCE -**Supported distros:** ubuntu, debian +LOG = logging.getLogger(__name__) -**Config keys**:: +# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') +ADD_APT_REPO_MATCH = r"^[\w-]+:\w" - apt: - preserve_sources_list: - disable_suites: +frequency = PER_INSTANCE +distros = ["ubuntu", "debian"] +mirror_property = { + 'type': 'array', + 'item': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['arches'], + 'properties': { + 'arches': { + 'type': 'array', + 'item': { + 'type': 'string' + }, + 'minItems': 1 + }, + 'uri': { + 'type': 'string', + 'format': 'uri' + }, + 'search': { + 'type': 'array', + 'item': { + 'type': 'string', + 'format': 'uri' + }, + 'minItems': 1 + }, + 'search_dns': { + 'type': 'boolean', + } + } + } +} +schema = { + 'id': 'cc_apt_configure', + 'name': 'Apt Configure', + 'title': 'Configure apt for the user', + 'description': dedent("""\ + This module handles both configuration of apt options and adding + source lists. There are configuration options such as + ``apt_get_wrapper`` and ``apt_get_command`` that control how + cloud-init invokes apt-get. These configuration options are + handled on a per-distro basis, so consult documentation for + cloud-init's distro support for instructions on using + these config options. + + .. note:: + To ensure that apt configuration is valid yaml, any strings + containing special characters, especially ``:`` should be quoted. + + .. note:: + For more information about apt configuration, see the + ``Additional apt configuration`` example."""), + 'distros': distros, + 'examples': [dedent("""\ + apt: + preserve_sources_list: false + disable_suites: - $RELEASE-updates - backports - $RELEASE - mysuite - primary: + primary: - arches: - amd64 - i386 - default - uri: "http://us.archive.ubuntu.com/ubuntu" + uri: 'http://us.archive.ubuntu.com/ubuntu' search: - - "http://cool.but-sometimes-unreachable.com/ubuntu" - - "http://us.archive.ubuntu.com/ubuntu" + - 'http://cool.but-sometimes-unreachable.com/ubuntu' + - 'http://us.archive.ubuntu.com/ubuntu' search_dns: - arches: - s390x - arm64 - uri: "http://archive-to-use-for-arm64.example.com/ubuntu" - security: + uri: 'http://archive-to-use-for-arm64.example.com/ubuntu' + security: - arches: - default search_dns: true - sources_list: | - deb $MIRROR $RELEASE main restricted - deb-src $MIRROR $RELEASE main restricted - deb $PRIMARY $RELEASE universe restricted - deb $SECURITY $RELEASE-security multiverse - debconf_selections: - set1: the-package the-package/some-flag boolean true - conf: | - APT { - Get { - Assume-Yes "true"; - Fix-Broken "true"; + sources_list: | + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + debconf_selections: + set1: the-package the-package/some-flag boolean true + conf: | + APT { + Get { + Assume-Yes 'true'; + Fix-Broken 'true'; + } + } + proxy: 'http://[[user][:pass]@]host[:port]/' + http_proxy: 'http://[[user][:pass]@]host[:port]/' + ftp_proxy: 'ftp://[[user][:pass]@]host[:port]/' + https_proxy: 'https://[[user][:pass]@]host[:port]/' + sources: + source1: + keyid: 'keyid' + keyserver: 'keyserverurl' + source: 'deb http:/// xenial main' + source2: + source: 'ppa:' + source3: + source: 'deb $MIRROR $RELEASE multiverse' + key: | + ------BEGIN PGP PUBLIC KEY BLOCK------- + + ------END PGP PUBLIC KEY BLOCK-------""")], + 'frequency': frequency, + 'type': 'object', + 'properties': { + 'apt': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'preserve_sources_list': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + By default, cloud-init will generate a new sources + list in ``/etc/apt/sources.list.d`` based on any + changes specified in cloud config. To disable this + behavior and preserve the sources list from the + pristine image, set ``preserve_sources_list`` + to ``true``. + + The ``preserve_sources_list`` option overrides + all other config keys that would alter + ``sources.list`` or ``sources.list.d``, + **except** for additional sources to be added + to ``sources.list.d``.""") + }, + 'disable_suites': { + 'type': 'array', + 'items': { + 'type': 'string' + }, + 'uniqueItems': True, + 'description': dedent("""\ + Entries in the sources list can be disabled using + ``disable_suites``, which takes a list of suites + to be disabled. If the string ``$RELEASE`` is + present in a suite in the ``disable_suites`` list, + it will be replaced with the release name. If a + suite specified in ``disable_suites`` is not + present in ``sources.list`` it will be ignored. + For convenience, several aliases are provided for + ``disable_suites``: + + - ``updates`` => ``$RELEASE-updates`` + - ``backports`` => ``$RELEASE-backports`` + - ``security`` => ``$RELEASE-security`` + - ``proposed`` => ``$RELEASE-proposed`` + - ``release`` => ``$RELEASE``. + + When a suite is disabled using ``disable_suites``, + its entry in ``sources.list`` is not deleted; it + is just commented out.""") + }, + 'primary': { + **mirror_property, + 'description': dedent("""\ + The primary and security archive mirrors can + be specified using the ``primary`` and + ``security`` keys, respectively. Both the + ``primary`` and ``security`` keys take a list + of configs, allowing mirrors to be specified + on a per-architecture basis. Each config is a + dictionary which must have an entry for + ``arches``, specifying which architectures + that config entry is for. The keyword + ``default`` applies to any architecture not + explicitly listed. The mirror url can be specified + with the ``uri`` key, or a list of mirrors to + check can be provided in order, with the first + mirror that can be resolved being selected. This + allows the same configuration to be used in + different environment, with different hosts used + for a local apt mirror. If no mirror is provided + by ``uri`` or ``search``, ``search_dns`` may be + used to search for dns names in the format + ``-mirror`` in each of the following: + + - fqdn of this host per cloud metadata, + - localdomain, + - domains listed in ``/etc/resolv.conf``. + + If there is a dns entry for ``-mirror``, + then it is assumed that there is a distro mirror + at ``http://-mirror./``. + If the ``primary`` key is defined, but not the + ``security`` key, then then configuration for + ``primary`` is also used for ``security``. + If ``search_dns`` is used for the ``security`` + key, the search pattern will be + ``-security-mirror``. + + If no mirrors are specified, or all lookups fail, + then default mirrors defined in the datasource + are used. If none are present in the datasource + either the following defaults are used: + + - ``primary`` => \ + ``http://archive.ubuntu.com/ubuntu``. + - ``security`` => \ + ``http://security.ubuntu.com/ubuntu`` + """)}, + 'security': { + **mirror_property, + 'description': dedent("""\ + Please refer to the primary config documentation""") + }, + 'add_apt_repo_match': { + 'type': 'string', + 'default': ADD_APT_REPO_MATCH, + 'description': dedent("""\ + All source entries in ``apt-sources`` that match + regex in ``add_apt_repo_match`` will be added to + the system using ``add-apt-repository``. If + ``add_apt_repo_match`` is not specified, it + defaults to ``{}``""".format(ADD_APT_REPO_MATCH)) + }, + 'debconf_selections': { + 'type': 'object', + 'items': {'type': 'string'}, + 'description': dedent("""\ + Debconf additional configurations can be specified as a + dictionary under the ``debconf_selections`` config + key, with each key in the dict representing a + different set of configurations. The value of each key + must be a string containing all the debconf + configurations that must be applied. We will bundle + all of the values and pass them to + ``debconf-set-selections``. Therefore, each value line + must be a valid entry for ``debconf-set-selections``, + meaning that they must possess for distinct fields: + + ``pkgname question type answer`` + + Where: + + - ``pkgname`` is the name of the package. + - ``question`` the name of the questions. + - ``type`` is the type of question. + - ``answer`` is the value used to ansert the \ + question. + + For example: \ + ``ippackage ippackage/ip string 127.0.01`` + """) + }, + 'sources_list': { + 'type': 'string', + 'description': dedent("""\ + Specifies a custom template for rendering + ``sources.list`` . If no ``sources_list`` template + is given, cloud-init will use sane default. Within + this template, the following strings will be + replaced with the appropriate values: + + - ``$MIRROR`` + - ``$RELEASE`` + - ``$PRIMARY`` + - ``$SECURITY``""") + }, + 'conf': { + 'type': 'string', + 'description': dedent("""\ + Specify configuration for apt, such as proxy + configuration. This configuration is specified as a + string. For multiline apt configuration, make sure + to follow yaml syntax.""") + }, + 'https_proxy': { + 'type': 'string', + 'description': dedent("""\ + More convenient way to specify https apt proxy. + https proxy url is specified in the format + ``https://[[user][:pass]@]host[:port]/``.""") + }, + 'http_proxy': { + 'type': 'string', + 'description': dedent("""\ + More convenient way to specify http apt proxy. + http proxy url is specified in the format + ``http://[[user][:pass]@]host[:port]/``.""") + }, + 'proxy': { + 'type': 'string', + 'description': 'Alias for defining a http apt proxy.' + }, + 'ftp_proxy': { + 'type': 'string', + 'description': dedent("""\ + More convenient way to specify ftp apt proxy. + ftp proxy url is specified in the format + ``ftp://[[user][:pass]@]host[:port]/``.""") + }, + 'sources': { + 'type': 'object', + 'items': {'type': 'string'}, + 'description': dedent("""\ + Source list entries can be specified as a + dictionary under the ``sources`` config key, with + each key in the dict representing a different source + file. The key of each source entry will be used + as an id that can be referenced in other config + entries, as well as the filename for the source's + configuration under ``/etc/apt/sources.list.d``. + If the name does not end with ``.list``, it will + be appended. If there is no configuration for a + key in ``sources``, no file will be written, but + the key may still be referred to as an id in other + ``sources`` entries. + + Each entry under ``sources`` is a dictionary which + may contain any of the following optional keys: + + - ``source``: a sources.list entry \ + (some variable replacements apply). + - ``keyid``: a key to import via shortid or \ + fingerprint. + - ``key``: a raw PGP key. + - ``keyserver``: alternate keyserver to pull \ + ``keyid`` key from. + + The ``source`` key supports variable + replacements for the following strings: + + - ``$MIRROR`` + - ``$PRIMARY`` + - ``$SECURITY`` + - ``$RELEASE``""") } } - proxy: "http://[[user][:pass]@]host[:port]/" - http_proxy: "http://[[user][:pass]@]host[:port]/" - ftp_proxy: "ftp://[[user][:pass]@]host[:port]/" - https_proxy: "https://[[user][:pass]@]host[:port]/" - sources: - source1: - keyid: "keyid" - keyserver: "keyserverurl" - source: "deb http:/// xenial main" - source2: - source: "ppa:" - source3: - source: "deb $MIRROR $RELEASE multiverse" - key: | - ------BEGIN PGP PUBLIC KEY BLOCK------- - - ------END PGP PUBLIC KEY BLOCK------- -""" - -import glob -import os -import re - -from cloudinit import gpg -from cloudinit import log as logging -from cloudinit import templater -from cloudinit import util + } + } +} -LOG = logging.getLogger(__name__) +__doc__ = get_schema_doc(schema) -# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') -ADD_APT_REPO_MATCH = r"^[\w-]+:\w" # place where apt stores cached repository data APT_LISTS = "/var/lib/apt/lists" @@ -279,6 +422,7 @@ def handle(name, ocfg, cloud, log, _): "Expected dictionary for 'apt' config, found {config_type}".format( config_type=type(cfg))) + validate_cloudconfig_schema(cfg, schema) apply_debconf_selections(cfg, target) apply_apt(cfg, cloud, target) diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 5498bbaa..3b2c2020 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -169,8 +169,8 @@ schema = { 'uniqueItems': True, 'description': dedent("""\ List of ntp pools. If both pools and servers are - empty, 4 default pool servers will be provided of - the format ``{0-3}.{distro}.pool.ntp.org``.""") + empty, 4 default pool servers will be provided of + the format ``{0-3}.{distro}.pool.ntp.org``.""") }, 'servers': { 'type': 'array', @@ -181,46 +181,46 @@ schema = { 'uniqueItems': True, 'description': dedent("""\ List of ntp servers. If both pools and servers are - empty, 4 default pool servers will be provided with - the format ``{0-3}.{distro}.pool.ntp.org``.""") + empty, 4 default pool servers will be provided with + the format ``{0-3}.{distro}.pool.ntp.org``.""") }, 'ntp_client': { 'type': 'string', 'default': 'auto', 'description': dedent("""\ Name of an NTP client to use to configure system NTP. - When unprovided or 'auto' the default client preferred - by the distribution will be used. The following - built-in client names can be used to override existing - configuration defaults: chrony, ntp, ntpdate, - systemd-timesyncd."""), + When unprovided or 'auto' the default client preferred + by the distribution will be used. The following + built-in client names can be used to override existing + configuration defaults: chrony, ntp, ntpdate, + systemd-timesyncd."""), }, 'enabled': { 'type': 'boolean', 'default': True, 'description': dedent("""\ Attempt to enable ntp clients if set to True. If set - to False, ntp client will not be configured or - installed"""), + to False, ntp client will not be configured or + installed"""), }, 'config': { 'description': dedent("""\ Configuration settings or overrides for the - ``ntp_client`` specified."""), + ``ntp_client`` specified."""), 'type': ['object'], 'properties': { 'confpath': { 'type': 'string', 'description': dedent("""\ The path to where the ``ntp_client`` - configuration is written."""), + configuration is written."""), }, 'check_exe': { 'type': 'string', 'description': dedent("""\ The executable name for the ``ntp_client``. - For example, ntp service ``check_exe`` is - 'ntpd' because it runs the ntpd binary."""), + For example, ntp service ``check_exe`` is + 'ntpd' because it runs the ntpd binary."""), }, 'packages': { 'type': 'array', @@ -230,22 +230,22 @@ schema = { 'uniqueItems': True, 'description': dedent("""\ List of packages needed to be installed for the - selected ``ntp_client``."""), + selected ``ntp_client``."""), }, 'service_name': { 'type': 'string', 'description': dedent("""\ The systemd or sysvinit service name used to - start and stop the ``ntp_client`` - service."""), + start and stop the ``ntp_client`` + service."""), }, 'template': { 'type': 'string', 'description': dedent("""\ Inline template allowing users to define their - own ``ntp_client`` configuration template. - The value must start with '## template:jinja' - to enable use of templating support. + own ``ntp_client`` configuration template. + The value must start with '## template:jinja' + to enable use of templating support. """), }, }, diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 204cbfd6..8601e707 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -103,7 +103,7 @@ schema = { 'type': 'string', 'description': dedent("""\ Path of the file to which ``content`` is decoded - and written + and written """), }, 'content': { @@ -111,9 +111,9 @@ schema = { 'default': '', 'description': dedent("""\ Optional content to write to the provided ``path``. - When content is present and encoding is not '%s', - decode the content prior to writing. Default: - **''** + When content is present and encoding is not '%s', + decode the content prior to writing. Default: + **''** """ % UNKNOWN_ENC), }, 'owner': { @@ -121,7 +121,7 @@ schema = { 'default': DEFAULT_OWNER, 'description': dedent("""\ Optional owner:group to chown on the file. Default: - **{owner}** + **{owner}** """.format(owner=DEFAULT_OWNER)), }, 'permissions': { @@ -129,8 +129,8 @@ schema = { 'default': oct(DEFAULT_PERMS).replace('o', ''), 'description': dedent("""\ Optional file permissions to set on ``path`` - represented as an octal string '0###'. Default: - **'{perms}'** + represented as an octal string '0###'. Default: + **'{perms}'** """.format(perms=oct(DEFAULT_PERMS).replace('o', ''))), }, 'encoding': { @@ -139,16 +139,16 @@ schema = { 'enum': supported_encoding_types, 'description': dedent("""\ Optional encoding type of the content. Default is - **text/plain** and no content decoding is - performed. Supported encoding types are: - %s.""" % ", ".join(supported_encoding_types)), + **text/plain** and no content decoding is + performed. Supported encoding types are: + %s.""" % ", ".join(supported_encoding_types)), }, 'append': { 'type': 'boolean', 'default': False, 'description': dedent("""\ Whether to append ``content`` to existing file if - ``path`` exists. Default: **false**. + ``path`` exists. Default: **false**. """), }, }, diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index a295d63d..da00060a 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -306,6 +306,28 @@ def _get_property_type(property_dict): return property_type +def _parse_description(description, prefix): + """Parse description from the schema in a format that we can better + display in our docs. This parser does three things: + + - Guarantee that a paragraph will be in a single line + - Guarantee that each new paragraph will be aligned with + the first paragraph + - Proper align lists of items + + @param description: The original description in the schema. + @param prefix: The number of spaces used to align the current description + """ + list_paragraph = prefix * 3 + description = re.sub(r"(\S)\n(\S)", r"\1 \2", description) + description = re.sub( + r"\n\n", r"\n\n{}".format(prefix), description) + description = re.sub( + r"\n( +)-", r"\n{}-".format(list_paragraph), description) + + return description + + def _get_property_doc(schema, prefix=' '): """Return restructured text describing the supported schema properties.""" new_prefix = prefix + ' ' @@ -318,7 +340,7 @@ def _get_property_doc(schema, prefix=' '): prefix=prefix, prop_name=prop_key, type=_get_property_type(prop_config), - description=description.replace('\n', ''))) + description=_parse_description(description, prefix))) items = prop_config.get('items') if items: if isinstance(items, list): diff --git a/doc/examples/cloud-config-apt.txt b/doc/examples/cloud-config-apt.txt index 5a56cd1b..004894b7 100644 --- a/doc/examples/cloud-config-apt.txt +++ b/doc/examples/cloud-config-apt.txt @@ -142,7 +142,7 @@ apt: # as above, allowing to have one config for different per arch mirrors # security is optional, if not defined it is set to the same value as primary security: - uri: http://security.ubuntu.com/ubuntu + - uri: http://security.ubuntu.com/ubuntu # If search_dns is set for security the searched pattern is: # -security-mirror diff --git a/doc/examples/cloud-config-chef-oneiric.txt b/doc/examples/cloud-config-chef-oneiric.txt index ab2e2a33..241fbf9b 100644 --- a/doc/examples/cloud-config-chef-oneiric.txt +++ b/doc/examples/cloud-config-chef-oneiric.txt @@ -13,38 +13,39 @@ # Key from http://apt.opscode.com/packages@opscode.com.gpg.key apt: sources: - - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1.4.9 (GNU/Linux) + source1: + source: "deb http://apt.opscode.com/ $RELEASE-0.10 main" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1.4.9 (GNU/Linux) - mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu - twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 - dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC - JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W - ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I - XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe - DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm - sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO - Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ - YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG - CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K - +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR - lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh - DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu - wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx - EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g - w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8 - AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN - QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X - Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ - 3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V - Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL - zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb - DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG - 0GLl8EkfA8uhluM= - =zKAm - -----END PGP PUBLIC KEY BLOCK----- + mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu + twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 + dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC + JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W + ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I + XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe + DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm + sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO + Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ + YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG + CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K + +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR + lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh + DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu + wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx + EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g + w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8 + AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN + QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X + Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ + 3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V + Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL + zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb + DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG + 0GLl8EkfA8uhluM= + =zKAm + -----END PGP PUBLIC KEY BLOCK----- chef: diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 8c1e4bb0..20a0ce0d 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -203,13 +203,14 @@ ssh_import_id: [smoser] # # Default: none # -debconf_selections: | # Need to preserve newlines +debconf_selections: # Force debconf priority to critical. - debconf debconf/priority select critical + set1: debconf debconf/priority select critical # Override default frontend to readline, but allow user to select. - debconf debconf/frontend select readline - debconf debconf/frontend seen false + set2: | + debconf debconf/frontend select readline + debconf debconf/frontend seen false # manage byobu defaults # byobu_by_default: diff --git a/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml b/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml index 0bec305e..68ca95b5 100644 --- a/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml +++ b/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml @@ -8,43 +8,44 @@ cloud_config: | #cloud-config # Key from https://packages.chef.io/chef.asc apt: - source1: - source: "deb http://packages.chef.io/repos/apt/stable $RELEASE main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1.4.12 (Darwin) - Comment: GPGTools - http://gpgtools.org + sources: + source1: + source: "deb http://packages.chef.io/repos/apt/stable $RELEASE main" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1.4.12 (Darwin) + Comment: GPGTools - http://gpgtools.org - mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu - twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 - dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC - JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W - ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I - XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe - DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm - sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO - Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ - YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG - CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K - +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu0IENIRUYgUGFja2FnZXMg - PHBhY2thZ2VzQGNoZWYuaW8+iGIEExECACIFAlQwYFECGwMGCwkIBwMCBhUIAgkK - CwQWAgMBAh4BAheAAAoJEClAq6mD74JqX94An26z99XOHWpLN8ahzm7cp13t4Xid - AJ9wVcgoUBzvgg91lKfv/34cmemZn7kCDQRKaQu0EAgAg7ZLCVGVTmLqBM6njZEd - Zbv+mZbvwLBSomdiqddE6u3eH0X3GuwaQfQWHUVG2yedyDMiG+EMtCdEeeRebTCz - SNXQ8Xvi22hRPoEsBSwWLZI8/XNg0n0f1+GEr+mOKO0BxDB2DG7DA0nnEISxwFkK - OFJFebR3fRsrWjj0KjDxkhse2ddU/jVz1BY7Nf8toZmwpBmdozETMOTx3LJy1HZ/ - Te9FJXJMUaB2lRyluv15MVWCKQJro4MQG/7QGcIfrIZNfAGJ32DDSjV7/YO+IpRY - IL4CUBQ65suY4gYUG4jhRH6u7H1p99sdwsg5OIpBe/v2Vbc/tbwAB+eJJAp89Zeu - twADBQf/ZcGoPhTGFuzbkcNRSIz+boaeWPoSxK2DyfScyCAuG41CY9+g0HIw9Sq8 - DuxQvJ+vrEJjNvNE3EAEdKl/zkXMZDb1EXjGwDi845TxEMhhD1dDw2qpHqnJ2mtE - WpZ7juGwA3sGhi6FapO04tIGacCfNNHmlRGipyq5ZiKIRq9mLEndlECr8cwaKgkS - 0wWu+xmMZe7N5/t/TK19HXNh4tVacv0F3fYK54GUjt2FjCQV75USnmNY4KPTYLXA - dzC364hEMlXpN21siIFgB04w+TXn5UF3B4FfAy5hevvr4DtV4MvMiGLu0oWjpaLC - MpmrR3Ny2wkmO0h+vgri9uIP06ODWIhJBBgRAgAJBQJKaQu0AhsMAAoJEClAq6mD - 74Jq4hIAoJ5KrYS8kCwj26SAGzglwggpvt3CAJ0bekyky56vNqoegB+y4PQVDv4K - zA== - =IxPr - -----END PGP PUBLIC KEY BLOCK----- + mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu + twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 + dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC + JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W + ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I + XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe + DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm + sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO + Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ + YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG + CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K + +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu0IENIRUYgUGFja2FnZXMg + PHBhY2thZ2VzQGNoZWYuaW8+iGIEExECACIFAlQwYFECGwMGCwkIBwMCBhUIAgkK + CwQWAgMBAh4BAheAAAoJEClAq6mD74JqX94An26z99XOHWpLN8ahzm7cp13t4Xid + AJ9wVcgoUBzvgg91lKfv/34cmemZn7kCDQRKaQu0EAgAg7ZLCVGVTmLqBM6njZEd + Zbv+mZbvwLBSomdiqddE6u3eH0X3GuwaQfQWHUVG2yedyDMiG+EMtCdEeeRebTCz + SNXQ8Xvi22hRPoEsBSwWLZI8/XNg0n0f1+GEr+mOKO0BxDB2DG7DA0nnEISxwFkK + OFJFebR3fRsrWjj0KjDxkhse2ddU/jVz1BY7Nf8toZmwpBmdozETMOTx3LJy1HZ/ + Te9FJXJMUaB2lRyluv15MVWCKQJro4MQG/7QGcIfrIZNfAGJ32DDSjV7/YO+IpRY + IL4CUBQ65suY4gYUG4jhRH6u7H1p99sdwsg5OIpBe/v2Vbc/tbwAB+eJJAp89Zeu + twADBQf/ZcGoPhTGFuzbkcNRSIz+boaeWPoSxK2DyfScyCAuG41CY9+g0HIw9Sq8 + DuxQvJ+vrEJjNvNE3EAEdKl/zkXMZDb1EXjGwDi845TxEMhhD1dDw2qpHqnJ2mtE + WpZ7juGwA3sGhi6FapO04tIGacCfNNHmlRGipyq5ZiKIRq9mLEndlECr8cwaKgkS + 0wWu+xmMZe7N5/t/TK19HXNh4tVacv0F3fYK54GUjt2FjCQV75USnmNY4KPTYLXA + dzC364hEMlXpN21siIFgB04w+TXn5UF3B4FfAy5hevvr4DtV4MvMiGLu0oWjpaLC + MpmrR3Ny2wkmO0h+vgri9uIP06ODWIhJBBgRAgAJBQJKaQu0AhsMAAoJEClAq6mD + 74Jq4hIAoJ5KrYS8kCwj26SAGzglwggpvt3CAJ0bekyky56vNqoegB+y4PQVDv4K + zA== + =IxPr + -----END PGP PUBLIC KEY BLOCK----- chef: diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 071b98bc..e19d13b8 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -24,6 +24,7 @@ class GetSchemaTest(CiTestCase): schema = get_schema() self.assertCountEqual( [ + 'cc_apt_configure', 'cc_bootcmd', 'cc_locale', 'cc_ntp', @@ -289,6 +290,41 @@ class GetSchemaDocTest(CiTestCase): """), get_schema_doc(full_schema)) + def test_get_schema_doc_properly_parse_description(self): + """get_schema_doc description properly formatted""" + full_schema = copy(self.required_schema) + full_schema.update( + {'properties': { + 'p1': { + 'type': 'string', + 'description': dedent("""\ + This item + has the + following options: + + - option1 + - option2 + - option3 + + The default value is + option1""") + } + }} + ) + + self.assertIn( + dedent(""" + **Config schema**: + **p1:** (string) This item has the following options: + + - option1 + - option2 + - option3 + + The default value is option1 + """), + get_schema_doc(full_schema)) + def test_get_schema_doc_raises_key_errors(self): """get_schema_doc raises KeyErrors on missing keys.""" for key in self.required_schema: -- cgit v1.2.3 From 4261ae538563d262bc76b8c55f7cc0c8abb14b00 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 14 May 2020 11:25:02 -0400 Subject: cloudinit: remove unneeded __future__ imports (#362) We live in the future now. --- cloudinit/config/schema.py | 2 -- cloudinit/serial.py | 2 -- cloudinit/sources/DataSourceMAAS.py | 2 -- cloudinit/tests/helpers.py | 2 -- tests/unittests/test_cs_util.py | 2 -- tests/unittests/test_datasource/test_smartos.py | 2 -- tests/unittests/test_templating.py | 2 -- tests/unittests/test_util.py | 2 -- 8 files changed, 16 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index da00060a..2d8c7577 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -1,8 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. """schema.py: Set of module functions for processing cloud-config schema.""" -from __future__ import print_function - from cloudinit import importer from cloudinit.util import find_modules, load_file diff --git a/cloudinit/serial.py b/cloudinit/serial.py index f9ef7acc..67486e09 100644 --- a/cloudinit/serial.py +++ b/cloudinit/serial.py @@ -1,7 +1,5 @@ # This file is part of cloud-init. See LICENSE file for license information. -from __future__ import absolute_import - try: from serial import Serial except ImportError: diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 517913aa..c80f70c2 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -6,8 +6,6 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from __future__ import print_function - import hashlib import os import time diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index b0af54c1..f3ab7e8c 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -1,7 +1,5 @@ # This file is part of cloud-init. See LICENSE file for license information. -from __future__ import print_function - import functools import httpretty import io diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py index 2a1095b9..bfd07ecf 100644 --- a/tests/unittests/test_cs_util.py +++ b/tests/unittests/test_cs_util.py @@ -1,7 +1,5 @@ # This file is part of cloud-init. See LICENSE file for license information. -from __future__ import print_function - from cloudinit.tests import helpers as test_helpers from cloudinit.cs_utils import Cepko diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 0f0b42bb..f1ab1d7a 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -12,8 +12,6 @@ order to validate return responses. ''' -from __future__ import print_function - from binascii import crc32 import json import multiprocessing diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index c36e6eb0..cba09830 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -4,8 +4,6 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from __future__ import print_function - from cloudinit.tests import helpers as test_helpers import textwrap diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index e45bedc1..4b439267 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -1,7 +1,5 @@ # This file is part of cloud-init. See LICENSE file for license information. -from __future__ import print_function - import io import json import logging -- cgit v1.2.3 From 09492b1d0f4b1826a7a98709c61d48bf14a1ec64 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 14 May 2020 15:39:19 -0400 Subject: cloudinit: minor pylint fixes (#360) We recently discovered that pylint is failing to report some errors when invoked across our entire codebase (see https://github.com/PyCQA/pylint/issues/3611). I've run pylint across every Python file under cloudinit/[0], and this commit fixes the issues so-discovered. [0] find cloudinit/ -name "*.py" | xargs -n 1 -t .tox/pylint/bin/python -m pylint --- cloudinit/config/tests/test_snap.py | 2 +- cloudinit/config/tests/test_ubuntu_drivers.py | 7 +++++++ cloudinit/sources/helpers/tests/test_netlink.py | 2 +- cloudinit/tests/test_conftest.py | 2 ++ 4 files changed, 11 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py index cbbb173d..70676942 100644 --- a/cloudinit/config/tests/test_snap.py +++ b/cloudinit/config/tests/test_snap.py @@ -345,7 +345,7 @@ class TestSchema(CiTestCase, SchemaTestCaseMixin): def test_duplicates_are_fine_array_array(self): """Duplicated commands array/array entries are allowed.""" self.assertSchemaValid( - {'commands': [["echo", "bye"], ["echo" "bye"]]}, + {'commands': [["echo", "bye"], ["echo", "bye"]]}, "command entries can be duplicate.") def test_duplicates_are_fine_array_string(self): diff --git a/cloudinit/config/tests/test_ubuntu_drivers.py b/cloudinit/config/tests/test_ubuntu_drivers.py index 46952692..0aec1265 100644 --- a/cloudinit/config/tests/test_ubuntu_drivers.py +++ b/cloudinit/config/tests/test_ubuntu_drivers.py @@ -16,6 +16,13 @@ OLD_UBUNTU_DRIVERS_ERROR_STDERR = ( "(choose from 'list', 'autoinstall', 'devices', 'debug')\n") +# The tests in this module call helper methods which are decorated with +# mock.patch. pylint doesn't understand that mock.patch passes parameters to +# the decorated function, so it incorrectly reports that we aren't passing +# values for all parameters. Instead of annotating every single call, we +# disable it for the entire module: +# pylint: disable=no-value-for-parameter + class AnyTempScriptAndDebconfFile(object): def __init__(self, tmp_dir, debconf_file): diff --git a/cloudinit/sources/helpers/tests/test_netlink.py b/cloudinit/sources/helpers/tests/test_netlink.py index c2898a16..58c3adc6 100644 --- a/cloudinit/sources/helpers/tests/test_netlink.py +++ b/cloudinit/sources/helpers/tests/test_netlink.py @@ -87,7 +87,7 @@ class TestParseNetlinkMessage(CiTestCase): data = None with self.assertRaises(AssertionError) as context: read_rta_oper_state(data) - self.assertTrue('data is none', str(context.exception)) + self.assertEqual('data is none', str(context.exception)) def test_read_invalid_rta_operstate_none(self): '''read_rta_oper_state returns none if operstate is none''' diff --git a/cloudinit/tests/test_conftest.py b/cloudinit/tests/test_conftest.py index 11b91327..b39637d8 100644 --- a/cloudinit/tests/test_conftest.py +++ b/cloudinit/tests/test_conftest.py @@ -13,6 +13,8 @@ class TestDisableSubpUsage: def test_typeerrors_on_incorrect_usage(self): with pytest.raises(TypeError): + # We are intentionally passing no value for a parameter, so: + # pylint: disable=no-value-for-parameter util.subp() @pytest.mark.parametrize('disable_subp_usage', [False], indirect=True) -- cgit v1.2.3 From 8bcf1c069d8ddf28898342e541254bdeb54445df Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 18 May 2020 13:30:27 -0400 Subject: cc_snap: apply validation to snap.commands properties (#364) Specifically, ensure that given values are either strings, or arrays of strings. --- cloudinit/config/cc_snap.py | 14 ++++++++++++++ cloudinit/config/tests/test_snap.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py index fbe7787a..3bf2e250 100644 --- a/cloudinit/config/cc_snap.py +++ b/cloudinit/config/cc_snap.py @@ -85,6 +85,14 @@ schema = { 01: ['snap', 'install', 'vlc'] 02: snap install vlc 03: 'snap install vlc' + """), dedent("""\ + # You can use a list of commands + snap: + commands: + - ['install', 'vlc'] + - ['snap', 'install', 'vlc'] + - snap install vlc + - 'snap install vlc' """)], 'frequency': PER_INSTANCE, 'type': 'object', @@ -110,6 +118,12 @@ schema = { 'additionalItems': False, # Reject non-string & non-list 'minItems': 1, 'minProperties': 1, + 'additionalProperties': { + 'oneOf': [ + {'type': 'string'}, + {'type': 'array', 'items': {'type': 'string'}}, + ], + }, }, 'squashfuse_in_container': { 'type': 'boolean' diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py index 70676942..2be30186 100644 --- a/cloudinit/config/tests/test_snap.py +++ b/cloudinit/config/tests/test_snap.py @@ -310,6 +310,38 @@ class TestSchema(CiTestCase, SchemaTestCaseMixin): {'snap': {'commands': {'01': 'also valid'}}}, schema) self.assertEqual('', self.logs.getvalue()) + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_when_commands_values_are_invalid_type(self, _): + """Warnings when snap:commands values are invalid type (e.g. int)""" + validate_cloudconfig_schema( + {'snap': {'commands': [123]}}, schema) + validate_cloudconfig_schema( + {'snap': {'commands': {'01': 123}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\n" + "snap.commands.0: 123 is not valid under any of the given" + " schemas\n" + "WARNING: Invalid config:\n" + "snap.commands.01: 123 is not valid under any of the given" + " schemas\n", + self.logs.getvalue()) + + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_when_commands_list_values_are_invalid_type(self, _): + """Warnings when snap:commands list values are wrong type (e.g. int)""" + validate_cloudconfig_schema( + {'snap': {'commands': [["snap", "install", 123]]}}, schema) + validate_cloudconfig_schema( + {'snap': {'commands': {'01': ["snap", "install", 123]}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\n" + "snap.commands.0: ['snap', 'install', 123] is not valid under any" + " of the given schemas\n", + "WARNING: Invalid config:\n" + "snap.commands.0: ['snap', 'install', 123] is not valid under any" + " of the given schemas\n", + self.logs.getvalue()) + @mock.patch('cloudinit.config.cc_snap.add_assertions') def test_warn_schema_assertions_is_not_list_or_dict(self, _): """Warn when snap:assertions config is not a list or dict.""" -- cgit v1.2.3 From fae90f14f0668a673e02bc7313bc84828e7cae71 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 19 May 2020 22:52:18 -0400 Subject: test_resolv_conf: refresh stale comment (#374) --- cloudinit/config/tests/test_resolv_conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/tests/test_resolv_conf.py b/cloudinit/config/tests/test_resolv_conf.py index f3d97b82..6546a0b5 100644 --- a/cloudinit/config/tests/test_resolv_conf.py +++ b/cloudinit/config/tests/test_resolv_conf.py @@ -35,8 +35,8 @@ class TestGenerateResolvConf: # Patch in templater so we can assert on the actual generated content @mock.patch("cloudinit.templater.util.write_file") # Parameterise with the value to be passed to generate_resolv_conf as the - # `params` parameter, and a list of the expected lines after the header as - # `extra_lines`. + # params parameter, and the expected line after the header as + # expected_extra_line. @pytest.mark.parametrize( "params,expected_extra_line", [ -- cgit v1.2.3 From 98a6d9a0da82a764943c9495d7f21f3acc5ccd05 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 21 May 2020 11:16:35 -0400 Subject: cc_snap: validate that assertions property values are strings (#370) And add an example of providing a list of assertions. --- cloudinit/config/cc_snap.py | 14 +++++++++----- cloudinit/config/tests/test_snap.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py index 3bf2e250..8178562e 100644 --- a/cloudinit/config/cc_snap.py +++ b/cloudinit/config/cc_snap.py @@ -93,6 +93,13 @@ schema = { - ['snap', 'install', 'vlc'] - snap install vlc - 'snap install vlc' + """), dedent("""\ + # You can use a list of assertions + snap: + assertions: + - signed_assertion_blob_here + - | + signed_assertion_blob_here """)], 'frequency': PER_INSTANCE, 'type': 'object', @@ -106,7 +113,8 @@ schema = { 'additionalItems': False, # Reject items non-string 'minItems': 1, 'minProperties': 1, - 'uniqueItems': True + 'uniqueItems': True, + 'additionalProperties': {'type': 'string'}, }, 'commands': { 'type': ['object', 'array'], # Array of strings or dict @@ -136,10 +144,6 @@ schema = { } } -# TODO schema for 'assertions' and 'commands' are too permissive at the moment. -# Once python-jsonschema supports schema draft 6 add support for arbitrary -# object keys with 'patternProperties' constraint to validate string values. - __doc__ = get_schema_doc(schema) # Supplement python help() SNAP_CMD = "snap" diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py index 2be30186..95270fa0 100644 --- a/cloudinit/config/tests/test_snap.py +++ b/cloudinit/config/tests/test_snap.py @@ -342,6 +342,20 @@ class TestSchema(CiTestCase, SchemaTestCaseMixin): " of the given schemas\n", self.logs.getvalue()) + @mock.patch('cloudinit.config.cc_snap.run_commands') + def test_schema_when_assertions_values_are_invalid_type(self, _): + """Warnings when snap:assertions values are invalid type (e.g. int)""" + validate_cloudconfig_schema( + {'snap': {'assertions': [123]}}, schema) + validate_cloudconfig_schema( + {'snap': {'assertions': {'01': 123}}}, schema) + self.assertEqual( + "WARNING: Invalid config:\n" + "snap.assertions.0: 123 is not of type 'string'\n" + "WARNING: Invalid config:\n" + "snap.assertions.01: 123 is not of type 'string'\n", + self.logs.getvalue()) + @mock.patch('cloudinit.config.cc_snap.add_assertions') def test_warn_schema_assertions_is_not_list_or_dict(self, _): """Warn when snap:assertions config is not a list or dict.""" -- cgit v1.2.3 From 0919bd46bbd1b12158c369569ec1298bb000dd8a Mon Sep 17 00:00:00 2001 From: Bipin Bachhao Date: Wed, 27 May 2020 18:49:38 -0400 Subject: Enable chef_license support for chef infra client (#389) Co-authored-by: Daniel Watkins --- cloudinit/config/cc_chef.py | 2 ++ doc/examples/cloud-config-chef.txt | 3 +++ templates/chef_client.rb.tmpl | 3 +++ tests/unittests/test_handler/test_handler_chef.py | 2 ++ 4 files changed, 10 insertions(+) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 01d61fa1..03285ef0 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -50,6 +50,7 @@ file). written to /etc/chef/client.rb) chef: + chef_license: client_key: encrypted_data_bag_secret: environment: @@ -125,6 +126,7 @@ CHEF_RB_TPL_PATH_KEYS = frozenset([ 'file_cache_path', 'pid_file', 'encrypted_data_bag_secret', + 'chef_license', ]) CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt index 39957e58..bb4b058c 100644 --- a/doc/examples/cloud-config-chef.txt +++ b/doc/examples/cloud-config-chef.txt @@ -52,6 +52,9 @@ apt: chef: + # Valid values are 'accept' and 'accept-no-persist' + chef_license: "accept" + # Valid values are 'gems' and 'packages' and 'omnibus' install_type: "packages" diff --git a/templates/chef_client.rb.tmpl b/templates/chef_client.rb.tmpl index 99978d3b..0a759b04 100644 --- a/templates/chef_client.rb.tmpl +++ b/templates/chef_client.rb.tmpl @@ -14,6 +14,9 @@ you need to add the following to config: The reason these are not in quotes is because they are ruby symbols that will be placed inside here, and not actual strings... #} +{% if chef_license %} +chef_license "{{chef_license}}" +{% endif%} {% if log_level %} log_level {{log_level}} {% endif %} diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 513c18b5..8c476418 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -130,6 +130,7 @@ class TestChef(FilesystemMockingTestCase): # This should create a file of the format... # Created by cloud-init v. 0.7.6 on Sat, 11 Oct 2014 23:57:21 +0000 + chef_license "accept" log_level :info ssl_verify_mode :verify_none log_location "/var/log/chef/client.log" @@ -153,6 +154,7 @@ class TestChef(FilesystemMockingTestCase): util.write_file('/etc/cloud/templates/chef_client.rb.tmpl', tpl_file) cfg = { 'chef': { + 'chef_license': "accept", 'server_url': 'localhost', 'validation_name': 'bob', 'validation_key': "/etc/chef/vkey.pem", -- cgit v1.2.3 From fc07d633f7cb694423349a2c4b10c91c4b4981a2 Mon Sep 17 00:00:00 2001 From: Matthew Ruffell Date: Tue, 2 Jun 2020 09:55:40 +1200 Subject: cc_grub_dpkg: determine idevs in more robust manner with grub-probe (#358) Replace the hardcoded list of devices with a more robust way of determining the device which grub is installed to. We use grub-probe to fetch the underlying disk the /boot directory is located on, and attempt to match the disk with its /dev/disk/by-id value. If no such /dev/disk/by-id/ value exists, we fallback to the plain disk name. The changes are robust to unstable kernel device names and ordering, and use /dev/disk/by-id values to populate grub-pc/install_devices where possible. LP: #1877491 --- cloudinit/config/cc_grub_dpkg.py | 95 ++++++++++++----- cloudinit/config/tests/test_grub_dpkg.py | 176 +++++++++++++++++++++++++++++++ tools/.github-cla-signers | 1 + 3 files changed, 247 insertions(+), 25 deletions(-) create mode 100644 cloudinit/config/tests/test_grub_dpkg.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index a323edfa..7888464e 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -1,8 +1,9 @@ -# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2009-2010, 2020 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser # Author: Juerg Haefliger +# Author: Matthew Ruffell # # This file is part of cloud-init. See LICENSE file for license information. @@ -15,15 +16,15 @@ Configure which device is used as the target for grub installation. This module should work correctly by default without any user configuration. It can be enabled/disabled using the ``enabled`` config key in the ``grub_dpkg`` config dict. The global config key ``grub-dpkg`` is an alias for ``grub_dpkg``. If no -installation device is specified this module will look for the first existing -device in: +installation device is specified this module will execute grub-probe to +determine which disk the /boot directory is associated with. - - ``/dev/sda`` - - ``/dev/vda`` - - ``/dev/xvda`` - - ``/dev/sda1`` - - ``/dev/vda1`` - - ``/dev/xvda1`` +The value which is placed into the debconf database is in the format which the +grub postinstall script expects. Normally, this is a /dev/disk/by-id/ value, +but we do fallback to the plain disk name if a by-id name is not present. + +If this module is executed inside a container, then the debconf database is +seeded with empty values, and install_devices_empty is set to true. **Internal name:** ``cc_grub_dpkg`` @@ -43,10 +44,66 @@ device in: import os from cloudinit import util +from cloudinit.util import ProcessExecutionError distros = ['ubuntu', 'debian'] +def fetch_idevs(log): + """ + Fetches the /dev/disk/by-id device grub is installed to. + Falls back to plain disk name if no by-id entry is present. + """ + disk = "" + devices = [] + + try: + # get the root disk where the /boot directory resides. + disk = util.subp(['grub-probe', '-t', 'disk', '/boot'], + capture=True)[0].strip() + except ProcessExecutionError as e: + # grub-common may not be installed, especially on containers + # FileNotFoundError is a nested exception of ProcessExecutionError + if isinstance(e.reason, FileNotFoundError): + log.debug("'grub-probe' not found in $PATH") + # disks from the container host are present in /proc and /sys + # which is where grub-probe determines where /boot is. + # it then checks for existence in /dev, which fails as host disks + # are not exposed to the container. + elif "failed to get canonical path" in e.stderr: + log.debug("grub-probe 'failed to get canonical path'") + else: + # something bad has happened, continue to log the error + raise + except Exception: + util.logexc(log, "grub-probe failed to execute for grub-dpkg") + + if not disk or not os.path.exists(disk): + # If we failed to detect a disk, we can return early + return '' + + try: + # check if disk exists and use udevadm to fetch symlinks + devices = util.subp( + ['udevadm', 'info', '--root', '--query=symlink', disk], + capture=True + )[0].strip().split() + except Exception: + util.logexc( + log, "udevadm DEVLINKS symlink query failed for disk='%s'", disk + ) + + log.debug('considering these device symlinks: %s', ','.join(devices)) + # filter symlinks for /dev/disk/by-id entries + devices = [dev for dev in devices if 'disk/by-id' in dev] + log.debug('filtered to these disk/by-id symlinks: %s', ','.join(devices)) + # select first device if there is one, else fall back to plain name + idevs = sorted(devices)[0] if devices else disk + log.debug('selected %s', idevs) + + return idevs + + def handle(name, cfg, _cloud, log, _args): mycfg = cfg.get("grub_dpkg", cfg.get("grub-dpkg", {})) @@ -62,22 +119,10 @@ def handle(name, cfg, _cloud, log, _args): idevs_empty = util.get_cfg_option_str( mycfg, "grub-pc/install_devices_empty", None) - if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): - if idevs is None: - idevs = "" - if idevs_empty is None: - idevs_empty = "true" - else: - if idevs_empty is None: - idevs_empty = "false" - if idevs is None: - idevs = "/dev/sda" - for dev in ("/dev/sda", "/dev/vda", "/dev/xvda", - "/dev/sda1", "/dev/vda1", "/dev/xvda1"): - if os.path.exists(dev): - idevs = dev - break + if idevs is None: + idevs = fetch_idevs(log) + if idevs_empty is None: + idevs_empty = "false" if idevs else "true" # now idevs and idevs_empty are set to determined values # or, those set by user diff --git a/cloudinit/config/tests/test_grub_dpkg.py b/cloudinit/config/tests/test_grub_dpkg.py new file mode 100644 index 00000000..01efa330 --- /dev/null +++ b/cloudinit/config/tests/test_grub_dpkg.py @@ -0,0 +1,176 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import pytest + +from unittest import mock +from logging import Logger +from cloudinit.util import ProcessExecutionError +from cloudinit.config.cc_grub_dpkg import fetch_idevs, handle + + +class TestFetchIdevs: + """Tests cc_grub_dpkg.fetch_idevs()""" + + # Note: udevadm info returns devices in a large single line string + @pytest.mark.parametrize( + "grub_output,path_exists,expected_log_call,udevadm_output" + ",expected_idevs", + [ + # Inside a container, grub not installed + ( + ProcessExecutionError(reason=FileNotFoundError()), + False, + mock.call("'grub-probe' not found in $PATH"), + '', + '', + ), + # Inside a container, grub installed + ( + ProcessExecutionError(stderr="failed to get canonical path"), + False, + mock.call("grub-probe 'failed to get canonical path'"), + '', + '', + ), + # KVM Instance + ( + ['/dev/vda'], + True, + None, + ( + '/dev/disk/by-path/pci-0000:00:00.0 ', + '/dev/disk/by-path/virtio-pci-0000:00:00.0 ' + ), + '/dev/vda', + ), + # Xen Instance + ( + ['/dev/xvda'], + True, + None, + '', + '/dev/xvda', + ), + # NVMe Hardware Instance + ( + ['/dev/nvme1n1'], + True, + None, + ( + '/dev/disk/by-id/nvme-Company_hash000 ', + '/dev/disk/by-id/nvme-nvme.000-000-000-000-000 ', + '/dev/disk/by-path/pci-0000:00:00.0-nvme-0 ' + ), + '/dev/disk/by-id/nvme-Company_hash000', + ), + # SCSI Hardware Instance + ( + ['/dev/sda'], + True, + None, + ( + '/dev/disk/by-id/company-user-1 ', + '/dev/disk/by-id/scsi-0Company_user-1 ', + '/dev/disk/by-path/pci-0000:00:00.0-scsi-0:0:0:0 ' + ), + '/dev/disk/by-id/company-user-1', + ), + ], + ) + @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc") + @mock.patch("cloudinit.config.cc_grub_dpkg.os.path.exists") + @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp") + def test_fetch_idevs(self, m_subp, m_exists, m_logexc, grub_output, + path_exists, expected_log_call, udevadm_output, + expected_idevs): + """Tests outputs from grub-probe and udevadm info against grub-dpkg""" + m_subp.side_effect = [ + grub_output, + ["".join(udevadm_output)] + ] + m_exists.return_value = path_exists + log = mock.Mock(spec=Logger) + idevs = fetch_idevs(log) + assert expected_idevs == idevs + if expected_log_call is not None: + assert expected_log_call in log.debug.call_args_list + + +class TestHandle: + """Tests cc_grub_dpkg.handle()""" + + @pytest.mark.parametrize( + "cfg_idevs,cfg_idevs_empty,fetch_idevs_output,expected_log_output", + [ + ( + # No configuration + None, + None, + '/dev/disk/by-id/nvme-Company_hash000', + ( + "Setting grub debconf-set-selections with ", + "'/dev/disk/by-id/nvme-Company_hash000','false'" + ), + ), + ( + # idevs set, idevs_empty unset + '/dev/sda', + None, + '/dev/sda', + ( + "Setting grub debconf-set-selections with ", + "'/dev/sda','false'" + ), + ), + ( + # idevs unset, idevs_empty set + None, + 'true', + '/dev/xvda', + ( + "Setting grub debconf-set-selections with ", + "'/dev/xvda','true'" + ), + ), + ( + # idevs set, idevs_empty set + '/dev/vda', + 'false', + '/dev/disk/by-id/company-user-1', + ( + "Setting grub debconf-set-selections with ", + "'/dev/vda','false'" + ), + ), + ( + # idevs set, idevs_empty set + # Respect what the user defines, even if its logically wrong + '/dev/nvme0n1', + 'true', + '', + ( + "Setting grub debconf-set-selections with ", + "'/dev/nvme0n1','true'" + ), + ) + ], + ) + @mock.patch("cloudinit.config.cc_grub_dpkg.fetch_idevs") + @mock.patch("cloudinit.config.cc_grub_dpkg.util.get_cfg_option_str") + @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc") + @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp") + def test_handle(self, m_subp, m_logexc, m_get_cfg_str, m_fetch_idevs, + cfg_idevs, cfg_idevs_empty, fetch_idevs_output, + expected_log_output): + """Test setting of correct debconf database entries""" + m_get_cfg_str.side_effect = [ + cfg_idevs, + cfg_idevs_empty + ] + m_fetch_idevs.return_value = fetch_idevs_output + log = mock.Mock(spec=Logger) + handle(mock.Mock(), mock.Mock(), mock.Mock(), log, mock.Mock()) + log.debug.assert_called_with("".join(expected_log_output)) + + +# vi: ts=4 expandtab diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 2c924f85..c3113705 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -2,6 +2,7 @@ beezly bipinbachhao dhensby lucasmoura +matthewruffell nishigori tomponline TheRealFalcon -- cgit v1.2.3 From 5f7825e22241423322dbe628de1b00289cf34114 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Tue, 2 Jun 2020 08:24:29 -0700 Subject: test: fix all flake8 E241 (#403) Remove extra spaces after a ',' --- cloudinit/cmd/tests/test_query.py | 2 +- cloudinit/config/cc_apt_configure.py | 2 +- cloudinit/net/tests/test_dhcp.py | 2 +- cloudinit/sources/DataSourceGCE.py | 2 +- cloudinit/tests/test_url_helper.py | 2 +- tests/unittests/test_net.py | 4 ++-- tox.ini | 3 +-- 7 files changed, 8 insertions(+), 9 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/cmd/tests/test_query.py b/cloudinit/cmd/tests/test_query.py index 6d36a4ea..cb15b2d2 100644 --- a/cloudinit/cmd/tests/test_query.py +++ b/cloudinit/cmd/tests/test_query.py @@ -261,7 +261,7 @@ class TestQuery(CiTestCase): args = self.args( debug=False, dump_all=False, format=None, instance_data=self.instance_data, list_keys=True, user_data='ud', - vendor_data='vd', varname='top') + vendor_data='vd', varname='top') with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: with mock.patch('os.getuid') as m_getuid: diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 9a33451d..b1c7b471 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -297,7 +297,7 @@ schema = { }, 'conf': { 'type': 'string', - 'description': dedent("""\ + 'description': dedent("""\ Specify configuration for apt, such as proxy configuration. This configuration is specified as a string. For multiline apt configuration, make sure diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index bc7bef45..7768da7c 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -211,7 +211,7 @@ class TestDHCPParseStaticRoutes(CiTestCase): "class_b": "16,172,16,10", "class_a": "8,10,10", "gateway": "0,0", - "netlen": "33,0", + "netlen": "33,0", } for rfc3442 in bad_rfc3442.values(): self.assertEqual([], parse_static_routes(rfc3442)) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 6cbfbbac..0ec5f6ec 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -116,7 +116,7 @@ def _write_host_key_to_guest_attributes(key_type, key_value): resp = url_helper.readurl(url=url, data=key_value, headers=HEADERS, request_method='PUT', check_status=False) if resp.ok(): - LOG.debug('Wrote %s host key to guest attributes.', key_type) + LOG.debug('Wrote %s host key to guest attributes.', key_type) else: LOG.debug('Unable to write %s host key to guest attributes.', key_type) diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py index 29b39374..364ec822 100644 --- a/cloudinit/tests/test_url_helper.py +++ b/cloudinit/tests/test_url_helper.py @@ -85,7 +85,7 @@ class TestReadFileOrUrl(CiTestCase): read_file_or_url(url, headers=headers, headers_redact=['sensitive']) logs = self.logs.getvalue() for k in headers.keys(): - self.assertEqual(headers[k], httpretty.last_request().headers[k]) + self.assertEqual(headers[k], httpretty.last_request().headers[k]) self.assertIn(REDACTED, logs) self.assertNotIn('sekret', logs) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index e075a64c..84d3a5f0 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -944,7 +944,7 @@ NETWORK_CONFIGS = { dhcp6: true """).rstrip(' '), 'expected_sysconfig_opensuse': { - 'ifcfg-iface0': textwrap.dedent("""\ + 'ifcfg-iface0': textwrap.dedent("""\ BOOTPROTO=dhcp DHCLIENT6_MODE=managed STARTMODE=auto""") @@ -1028,7 +1028,7 @@ NETWORK_CONFIGS = { }, 'v6_and_v4': { 'expected_sysconfig_opensuse': { - 'ifcfg-iface0': textwrap.dedent("""\ + 'ifcfg-iface0': textwrap.dedent("""\ BOOTPROTO=dhcp DHCLIENT6_MODE=managed STARTMODE=auto""") diff --git a/tox.ini b/tox.ini index 725a0ca4..5ee67368 100644 --- a/tox.ini +++ b/tox.ini @@ -47,10 +47,9 @@ deps = -r{toxinidir}/test-requirements.txt # E123: closing bracket does not match indentation of opening bracket’s line # E126: continuation line over-indented for hanging indent # E226: missing whitespace around arithmetic operator -# E241: multiple spaces after ‘,’ # W503: line break before binary operator # W504: line break after binary operator -ignore=E121,E123,E126,E226,E241,W503,W504 +ignore=E121,E123,E126,E226,W503,W504 exclude = .venv,.tox,dist,doc,*egg,.git,build,tools per-file-ignores = cloudinit/cmd/main.py:E402 -- cgit v1.2.3 From 3c551f6ebc12f7729a2755c89b19b9000e27cc88 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 8 Jun 2020 12:49:12 -0400 Subject: Move subp into its own module. (#416) This was painful, but it finishes a TODO from cloudinit/subp.py. It moves the following from util to subp: ProcessExecutionError subp which target_path I moved subp_blob_in_tempfile into cc_chef, which is its only caller. That saved us from having to deal with it using write_file and temp_utils from subp (which does not import any cloudinit things now). It is arguable that 'target_path' could be moved to a 'path_utils' or something, but in order to use it from subp and also from utils, we had to get it out of utils. --- cloudinit/analyze/dump.py | 3 +- cloudinit/analyze/show.py | 5 +- cloudinit/analyze/tests/test_boot.py | 16 +- cloudinit/analyze/tests/test_dump.py | 3 +- cloudinit/cmd/clean.py | 5 +- cloudinit/cmd/devel/logs.py | 4 +- cloudinit/cmd/devel/tests/test_logs.py | 3 +- cloudinit/config/cc_apt_configure.py | 21 +- cloudinit/config/cc_bootcmd.py | 3 +- cloudinit/config/cc_byobu.py | 3 +- cloudinit/config/cc_ca_certs.py | 5 +- cloudinit/config/cc_chef.py | 35 +- cloudinit/config/cc_disable_ec2_metadata.py | 7 +- cloudinit/config/cc_disk_setup.py | 45 +-- cloudinit/config/cc_emit_upstart.py | 8 +- cloudinit/config/cc_fan.py | 7 +- cloudinit/config/cc_growpart.py | 25 +- cloudinit/config/cc_grub_dpkg.py | 9 +- cloudinit/config/cc_keys_to_console.py | 3 +- cloudinit/config/cc_landscape.py | 3 +- cloudinit/config/cc_lxd.py | 23 +- cloudinit/config/cc_mcollective.py | 3 +- cloudinit/config/cc_mounts.py | 15 +- cloudinit/config/cc_ntp.py | 9 +- .../config/cc_package_update_upgrade_install.py | 3 +- cloudinit/config/cc_power_state_change.py | 3 +- cloudinit/config/cc_puppet.py | 9 +- cloudinit/config/cc_resizefs.py | 9 +- cloudinit/config/cc_rh_subscription.py | 19 +- cloudinit/config/cc_rsyslog.py | 5 +- cloudinit/config/cc_salt_minion.py | 4 +- cloudinit/config/cc_seed_random.py | 5 +- cloudinit/config/cc_set_passwords.py | 5 +- cloudinit/config/cc_snap.py | 7 +- cloudinit/config/cc_spacewalk.py | 8 +- cloudinit/config/cc_ssh.py | 7 +- cloudinit/config/cc_ssh_import_id.py | 5 +- cloudinit/config/cc_ubuntu_advantage.py | 11 +- cloudinit/config/cc_ubuntu_drivers.py | 9 +- .../config/tests/test_disable_ec2_metadata.py | 12 +- cloudinit/config/tests/test_grub_dpkg.py | 6 +- cloudinit/config/tests/test_mounts.py | 4 +- cloudinit/config/tests/test_set_passwords.py | 22 +- cloudinit/config/tests/test_snap.py | 12 +- cloudinit/config/tests/test_ubuntu_advantage.py | 28 +- cloudinit/config/tests/test_ubuntu_drivers.py | 26 +- cloudinit/conftest.py | 28 +- cloudinit/distros/__init__.py | 25 +- cloudinit/distros/arch.py | 17 +- cloudinit/distros/bsd.py | 7 +- cloudinit/distros/debian.py | 9 +- cloudinit/distros/freebsd.py | 13 +- cloudinit/distros/gentoo.py | 17 +- cloudinit/distros/netbsd.py | 11 +- cloudinit/distros/openbsd.py | 3 +- cloudinit/distros/opensuse.py | 7 +- cloudinit/distros/rhel.py | 9 +- cloudinit/gpg.py | 14 +- cloudinit/handlers/boot_hook.py | 5 +- cloudinit/handlers/upstart_job.py | 11 +- cloudinit/net/__init__.py | 37 +- cloudinit/net/bsd.py | 11 +- cloudinit/net/dhcp.py | 7 +- cloudinit/net/eni.py | 9 +- cloudinit/net/freebsd.py | 7 +- cloudinit/net/netbsd.py | 5 +- cloudinit/net/netplan.py | 17 +- cloudinit/net/openbsd.py | 7 +- cloudinit/net/sysconfig.py | 19 +- cloudinit/net/tests/test_dhcp.py | 12 +- cloudinit/net/tests/test_init.py | 5 +- cloudinit/netinfo.py | 27 +- cloudinit/sources/DataSourceAltCloud.py | 8 +- cloudinit/sources/DataSourceAzure.py | 21 +- cloudinit/sources/DataSourceCloudStack.py | 3 +- cloudinit/sources/DataSourceConfigDrive.py | 3 +- cloudinit/sources/DataSourceIBMCloud.py | 3 +- cloudinit/sources/DataSourceOVF.py | 7 +- cloudinit/sources/DataSourceOpenNebula.py | 5 +- cloudinit/sources/DataSourceRbxCloud.py | 7 +- cloudinit/sources/DataSourceSmartOS.py | 5 +- cloudinit/sources/helpers/azure.py | 17 +- cloudinit/sources/helpers/digitalocean.py | 9 +- cloudinit/sources/helpers/openstack.py | 3 +- .../helpers/vmware/imc/config_custom_script.py | 3 +- cloudinit/sources/helpers/vmware/imc/config_nic.py | 7 +- .../sources/helpers/vmware/imc/config_passwd.py | 7 +- .../sources/helpers/vmware/imc/guestcust_util.py | 10 +- cloudinit/subp.py | 301 +++++++++++++++- cloudinit/tests/helpers.py | 13 +- cloudinit/tests/test_conftest.py | 24 +- cloudinit/tests/test_gpg.py | 10 +- cloudinit/tests/test_netinfo.py | 40 +-- cloudinit/tests/test_subp.py | 227 ++++++++++++- cloudinit/tests/test_util.py | 11 +- cloudinit/util.py | 377 ++------------------- packages/bddeb | 9 +- packages/brpm | 5 +- tests/cloud_tests/bddeb.py | 6 +- tests/cloud_tests/platforms/lxd/image.py | 9 +- tests/cloud_tests/platforms/lxd/instance.py | 3 +- tests/cloud_tests/platforms/nocloudkvm/image.py | 12 +- tests/cloud_tests/platforms/nocloudkvm/instance.py | 10 +- tests/cloud_tests/platforms/nocloudkvm/platform.py | 5 +- tests/cloud_tests/platforms/platforms.py | 9 +- tests/cloud_tests/util.py | 7 +- tests/unittests/test_builtin_handlers.py | 3 +- tests/unittests/test_datasource/test_altcloud.py | 7 +- tests/unittests/test_datasource/test_azure.py | 30 +- .../unittests/test_datasource/test_azure_helper.py | 2 +- tests/unittests/test_datasource/test_cloudstack.py | 2 +- tests/unittests/test_datasource/test_ovf.py | 9 +- tests/unittests/test_datasource/test_rbx.py | 10 +- tests/unittests/test_datasource/test_smartos.py | 6 +- tests/unittests/test_distros/test_create_users.py | 8 +- tests/unittests/test_distros/test_debian.py | 2 +- tests/unittests/test_distros/test_freebsd.py | 4 +- tests/unittests/test_distros/test_generic.py | 4 +- tests/unittests/test_distros/test_netconfig.py | 3 +- .../test_distros/test_user_data_normalize.py | 6 +- tests/unittests/test_ds_identify.py | 5 +- .../test_handler_apt_configure_sources_list_v1.py | 5 +- .../test_handler_apt_configure_sources_list_v3.py | 5 +- .../test_handler/test_handler_apt_source_v1.py | 13 +- .../test_handler/test_handler_apt_source_v3.py | 21 +- .../unittests/test_handler/test_handler_bootcmd.py | 6 +- .../test_handler/test_handler_ca_certs.py | 5 +- tests/unittests/test_handler/test_handler_chef.py | 6 +- .../test_handler/test_handler_disk_setup.py | 8 +- .../test_handler/test_handler_growpart.py | 10 +- .../test_handler/test_handler_landscape.py | 6 +- .../unittests/test_handler/test_handler_locale.py | 2 +- tests/unittests/test_handler/test_handler_lxd.py | 28 +- .../test_handler/test_handler_mcollective.py | 7 +- .../unittests/test_handler/test_handler_mounts.py | 6 +- tests/unittests/test_handler/test_handler_ntp.py | 51 +-- .../unittests/test_handler/test_handler_puppet.py | 26 +- .../unittests/test_handler/test_handler_runcmd.py | 4 +- .../test_handler/test_handler_seed_random.py | 5 +- .../test_handler/test_handler_spacewalk.py | 20 +- tests/unittests/test_net.py | 33 +- tests/unittests/test_net_freebsd.py | 2 +- tests/unittests/test_render_cloudcfg.py | 7 +- tests/unittests/test_reporting_hyperv.py | 2 +- tests/unittests/test_rh_subscription.py | 18 +- tests/unittests/test_util.py | 258 ++------------ tests/unittests/test_vmware/test_guestcust_util.py | 22 +- tools/.github-cla-signers | 1 + 148 files changed, 1403 insertions(+), 1292 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/analyze/dump.py b/cloudinit/analyze/dump.py index 939c3126..62ad51fe 100644 --- a/cloudinit/analyze/dump.py +++ b/cloudinit/analyze/dump.py @@ -4,6 +4,7 @@ import calendar from datetime import datetime import sys +from cloudinit import subp from cloudinit import util stage_to_description = { @@ -51,7 +52,7 @@ def parse_timestamp(timestampstr): def parse_timestamp_from_date(timestampstr): - out, _ = util.subp(['date', '+%s.%3N', '-d', timestampstr]) + out, _ = subp.subp(['date', '+%s.%3N', '-d', timestampstr]) timestamp = out.strip() return float(timestamp) diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py index fb152b1d..cca1fa7f 100644 --- a/cloudinit/analyze/show.py +++ b/cloudinit/analyze/show.py @@ -11,6 +11,7 @@ import os import time import sys +from cloudinit import subp from cloudinit import util from cloudinit.distros import uses_systemd @@ -155,7 +156,7 @@ class SystemctlReader(object): :return: whether the subp call failed or not ''' try: - value, err = util.subp(self.args, capture=True) + value, err = subp.subp(self.args, capture=True) if err: return err self.epoch = value @@ -215,7 +216,7 @@ def gather_timestamps_using_dmesg(): with gather_timestamps_using_systemd ''' try: - data, _ = util.subp(['dmesg'], capture=True) + data, _ = subp.subp(['dmesg'], capture=True) split_entries = data[0].splitlines() for i in split_entries: if i.decode('UTF-8').find('user') != -1: diff --git a/cloudinit/analyze/tests/test_boot.py b/cloudinit/analyze/tests/test_boot.py index f4001c14..f69423c3 100644 --- a/cloudinit/analyze/tests/test_boot.py +++ b/cloudinit/analyze/tests/test_boot.py @@ -25,7 +25,7 @@ class TestDistroChecker(CiTestCase): m_get_linux_distro, m_is_FreeBSD): self.assertEqual(err_code, dist_check_timestamp()) - @mock.patch('cloudinit.util.subp', return_value=(0, 1)) + @mock.patch('cloudinit.subp.subp', return_value=(0, 1)) def test_subp_fails(self, m_subp): self.assertEqual(err_code, dist_check_timestamp()) @@ -42,7 +42,7 @@ class TestSystemCtlReader(CiTestCase): with self.assertRaises(RuntimeError): reader.parse_epoch_as_float() - @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None)) def test_systemctl_works_correctly_threshold(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') self.assertEqual(1.0, reader.parse_epoch_as_float()) @@ -50,12 +50,12 @@ class TestSystemCtlReader(CiTestCase): self.assertTrue(thresh < 1e-6) self.assertTrue(thresh > (-1 * 1e-6)) - @mock.patch('cloudinit.util.subp', return_value=('U=0', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=0', None)) def test_systemctl_succeed_zero(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') self.assertEqual(0.0, reader.parse_epoch_as_float()) - @mock.patch('cloudinit.util.subp', return_value=('U=1', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1', None)) def test_systemctl_succeed_distinct(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') val1 = reader.parse_epoch_as_float() @@ -64,13 +64,13 @@ class TestSystemCtlReader(CiTestCase): val2 = reader2.parse_epoch_as_float() self.assertNotEqual(val1, val2) - @mock.patch('cloudinit.util.subp', return_value=('100', None)) + @mock.patch('cloudinit.subp.subp', return_value=('100', None)) def test_systemctl_epoch_not_splittable(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') with self.assertRaises(IndexError): reader.parse_epoch_as_float() - @mock.patch('cloudinit.util.subp', return_value=('U=foobar', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=foobar', None)) def test_systemctl_cannot_convert_epoch_to_float(self, m_subp): reader = SystemctlReader('dummyProperty', 'dummyParameter') with self.assertRaises(ValueError): @@ -130,7 +130,7 @@ class TestAnalyzeBoot(CiTestCase): self.assertEqual(err_string, data) @mock.patch("cloudinit.util.is_container", return_value=True) - @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None)) def test_container_no_ci_log_line(self, m_is_container, m_subp): path = os.path.dirname(os.path.abspath(__file__)) log_path = path + '/boot-test.log' @@ -148,7 +148,7 @@ class TestAnalyzeBoot(CiTestCase): self.assertEqual(FAIL_CODE, finish_code) @mock.patch("cloudinit.util.is_container", return_value=True) - @mock.patch('cloudinit.util.subp', return_value=('U=1000000', None)) + @mock.patch('cloudinit.subp.subp', return_value=('U=1000000', None)) @mock.patch('cloudinit.analyze.__main__._get_events', return_value=[{ 'name': 'init-local', 'description': 'starting search', 'timestamp': 100000}]) diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py index d6fbd381..dac1efb6 100644 --- a/cloudinit/analyze/tests/test_dump.py +++ b/cloudinit/analyze/tests/test_dump.py @@ -5,7 +5,8 @@ from textwrap import dedent from cloudinit.analyze.dump import ( dump_events, parse_ci_logline, parse_timestamp) -from cloudinit.util import which, write_file +from cloudinit.util import write_file +from cloudinit.subp import which from cloudinit.tests.helpers import CiTestCase, mock, skipIf diff --git a/cloudinit/cmd/clean.py b/cloudinit/cmd/clean.py index 30e49de0..928a8eea 100644 --- a/cloudinit/cmd/clean.py +++ b/cloudinit/cmd/clean.py @@ -10,9 +10,8 @@ import os import sys from cloudinit.stages import Init -from cloudinit.util import ( - ProcessExecutionError, del_dir, del_file, get_config_logfiles, - is_link, subp) +from cloudinit.subp import (ProcessExecutionError, subp) +from cloudinit.util import (del_dir, del_file, get_config_logfiles, is_link) def error(msg): diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py index 4c086b51..51c61cca 100644 --- a/cloudinit/cmd/devel/logs.py +++ b/cloudinit/cmd/devel/logs.py @@ -12,8 +12,8 @@ import sys from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE from cloudinit.temp_utils import tempdir -from cloudinit.util import ( - ProcessExecutionError, chdir, copy, ensure_dir, subp, write_file) +from cloudinit.subp import (ProcessExecutionError, subp) +from cloudinit.util import (chdir, copy, ensure_dir, write_file) CLOUDINIT_LOGS = ['/var/log/cloud-init.log', '/var/log/cloud-init-output.log'] diff --git a/cloudinit/cmd/devel/tests/test_logs.py b/cloudinit/cmd/devel/tests/test_logs.py index d2dfa8de..ddfd58e1 100644 --- a/cloudinit/cmd/devel/tests/test_logs.py +++ b/cloudinit/cmd/devel/tests/test_logs.py @@ -8,7 +8,8 @@ from cloudinit.cmd.devel import logs from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE from cloudinit.tests.helpers import ( FilesystemMockingTestCase, mock, wrap_and_call) -from cloudinit.util import ensure_dir, load_file, subp, write_file +from cloudinit.subp import subp +from cloudinit.util import ensure_dir, load_file, write_file @mock.patch('cloudinit.cmd.devel.logs.os.getuid') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index b1c7b471..73d8719f 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -17,6 +17,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit import gpg from cloudinit import log as logging +from cloudinit import subp from cloudinit import templater from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -431,7 +432,7 @@ def _should_configure_on_empty_apt(): # if no config was provided, should apt configuration be done? if util.system_is_snappy(): return False, "system is snappy." - if not (util.which('apt-get') or util.which('apt')): + if not (subp.which('apt-get') or subp.which('apt')): return False, "no apt commands." return True, "Apt is available." @@ -478,7 +479,7 @@ def apply_apt(cfg, cloud, target): def debconf_set_selections(selections, target=None): if not selections.endswith(b'\n'): selections += b'\n' - util.subp(['debconf-set-selections'], data=selections, target=target, + subp.subp(['debconf-set-selections'], data=selections, target=target, capture=True) @@ -503,7 +504,7 @@ def dpkg_reconfigure(packages, target=None): "but cannot be unconfigured: %s", unhandled) if len(to_config): - util.subp(['dpkg-reconfigure', '--frontend=noninteractive'] + + subp.subp(['dpkg-reconfigure', '--frontend=noninteractive'] + list(to_config), data=None, target=target, capture=True) @@ -546,7 +547,7 @@ def apply_debconf_selections(cfg, target=None): def clean_cloud_init(target): """clean out any local cloud-init config""" flist = glob.glob( - util.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) + subp.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) LOG.debug("cleaning cloud-init config from: %s", flist) for dpkg_cfg in flist: @@ -575,7 +576,7 @@ def rename_apt_lists(new_mirrors, target, arch): """rename_apt_lists - rename apt lists to preserve old cache data""" default_mirrors = get_default_mirrors(arch) - pre = util.target_path(target, APT_LISTS) + pre = subp.target_path(target, APT_LISTS) for (name, omirror) in default_mirrors.items(): nmirror = new_mirrors.get(name) if not nmirror: @@ -694,8 +695,8 @@ def add_apt_key_raw(key, target=None): """ LOG.debug("Adding key:\n'%s'", key) try: - util.subp(['apt-key', 'add', '-'], data=key.encode(), target=target) - except util.ProcessExecutionError: + subp.subp(['apt-key', 'add', '-'], data=key.encode(), target=target) + except subp.ProcessExecutionError: LOG.exception("failed to add apt GPG Key to apt keyring") raise @@ -758,13 +759,13 @@ def add_apt_sources(srcdict, cloud, target=None, template_params=None, if aa_repo_match(source): try: - util.subp(["add-apt-repository", source], target=target) - except util.ProcessExecutionError: + subp.subp(["add-apt-repository", source], target=target) + except subp.ProcessExecutionError: LOG.exception("add-apt-repository failed.") raise continue - sourcefn = util.target_path(target, ent['filename']) + sourcefn = subp.target_path(target, ent['filename']) try: contents = "%s\n" % (source) util.write_file(sourcefn, contents, omode="a") diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 6813f534..246e4497 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -16,6 +16,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_ALWAYS from cloudinit import temp_utils +from cloudinit import subp from cloudinit import util frequency = PER_ALWAYS @@ -99,7 +100,7 @@ def handle(name, cfg, cloud, log, _args): if iid: env['INSTANCE_ID'] = str(iid) cmd = ['/bin/sh', tmpf.name] - util.subp(cmd, env=env, capture=False) + subp.subp(cmd, env=env, capture=False) except Exception: util.logexc(log, "Failed to run bootcmd module %s", name) raise diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 0b4352c8..9fdaeba1 100755 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -39,6 +39,7 @@ Valid configuration options for this module are: """ from cloudinit.distros import ug_util +from cloudinit import subp from cloudinit import util distros = ['ubuntu', 'debian'] @@ -93,6 +94,6 @@ def handle(name, cfg, cloud, log, args): if len(shcmd): cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] log.debug("Setting byobu to %s", value) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 64bc900e..7617a8ea 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -36,6 +36,7 @@ can be removed from the system with the configuration option import os +from cloudinit import subp from cloudinit import util CA_CERT_PATH = "/usr/share/ca-certificates/" @@ -51,7 +52,7 @@ def update_ca_certs(): """ Updates the CA certificate cache on the current machine. """ - util.subp(["update-ca-certificates"], capture=False) + subp.subp(["update-ca-certificates"], capture=False) def add_ca_certs(certs): @@ -85,7 +86,7 @@ def remove_default_ca_certs(): util.delete_dir_contents(CA_CERT_SYSTEM_PATH) util.write_file(CA_CERT_CONFIG, "", mode=0o644) debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" - util.subp(('debconf-set-selections', '-'), debconf_sel) + subp.subp(('debconf-set-selections', '-'), debconf_sel) def handle(name, cfg, _cloud, log, _args): diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 03285ef0..e1f51fce 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -76,7 +76,9 @@ import itertools import json import os +from cloudinit import subp from cloudinit import templater +from cloudinit import temp_utils from cloudinit import url_helper from cloudinit import util @@ -282,7 +284,32 @@ def run_chef(chef_cfg, log): cmd.extend(CHEF_EXEC_DEF_ARGS) else: cmd.extend(CHEF_EXEC_DEF_ARGS) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) + + +def subp_blob_in_tempfile(blob, *args, **kwargs): + """Write blob to a tempfile, and call subp with args, kwargs. Then cleanup. + + 'basename' as a kwarg allows providing the basename for the file. + The 'args' argument to subp will be updated with the full path to the + filename as the first argument. + """ + basename = kwargs.pop('basename', "subp_blob") + + if len(args) == 0 and 'args' not in kwargs: + args = [tuple()] + + # Use tmpdir over tmpfile to avoid 'text file busy' on execute + with temp_utils.tempdir(needs_exe=True) as tmpd: + tmpf = os.path.join(tmpd, basename) + if 'args' in kwargs: + kwargs['args'] = [tmpf] + list(kwargs['args']) + else: + args = list(args) + args[0] = [tmpf] + args[0] + + util.write_file(tmpf, blob, mode=0o700) + return subp.subp(*args, **kwargs) def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None): @@ -305,7 +332,7 @@ def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None): else: args = ['-v', omnibus_version] content = url_helper.readurl(url=url, retries=retries).contents - return util.subp_blob_in_tempfile( + return subp_blob_in_tempfile( blob=content, args=args, basename='chef-omnibus-install', capture=False) @@ -354,11 +381,11 @@ def install_chef_from_gems(ruby_version, chef_version, distro): if not os.path.exists('/usr/bin/ruby'): util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') if chef_version: - util.subp(['/usr/bin/gem', 'install', 'chef', + subp.subp(['/usr/bin/gem', 'install', 'chef', '-v %s' % chef_version, '--no-ri', '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) else: - util.subp(['/usr/bin/gem', 'install', 'chef', + subp.subp(['/usr/bin/gem', 'install', 'chef', '--no-ri', '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) diff --git a/cloudinit/config/cc_disable_ec2_metadata.py b/cloudinit/config/cc_disable_ec2_metadata.py index 885b3138..dff93245 100644 --- a/cloudinit/config/cc_disable_ec2_metadata.py +++ b/cloudinit/config/cc_disable_ec2_metadata.py @@ -26,6 +26,7 @@ by default. disable_ec2_metadata: """ +from cloudinit import subp from cloudinit import util from cloudinit.settings import PER_ALWAYS @@ -40,15 +41,15 @@ def handle(name, cfg, _cloud, log, _args): disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False) if disabled: reject_cmd = None - if util.which('ip'): + if subp.which('ip'): reject_cmd = REJECT_CMD_IP - elif util.which('ifconfig'): + elif subp.which('ifconfig'): reject_cmd = REJECT_CMD_IF else: log.error(('Neither "route" nor "ip" command found, unable to ' 'manipulate routing table')) return - util.subp(reject_cmd, capture=False) + subp.subp(reject_cmd, capture=False) else: log.debug(("Skipping module named %s," " disabling the ec2 route not enabled"), name) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 45925755..d957cfe3 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -99,6 +99,7 @@ specified using ``filesystem``. from cloudinit.settings import PER_INSTANCE from cloudinit import util +from cloudinit import subp import logging import os import shlex @@ -106,13 +107,13 @@ import shlex frequency = PER_INSTANCE # Define the commands to use -UDEVADM_CMD = util.which('udevadm') -SFDISK_CMD = util.which("sfdisk") -SGDISK_CMD = util.which("sgdisk") -LSBLK_CMD = util.which("lsblk") -BLKID_CMD = util.which("blkid") -BLKDEV_CMD = util.which("blockdev") -WIPEFS_CMD = util.which("wipefs") +UDEVADM_CMD = subp.which('udevadm') +SFDISK_CMD = subp.which("sfdisk") +SGDISK_CMD = subp.which("sgdisk") +LSBLK_CMD = subp.which("lsblk") +BLKID_CMD = subp.which("blkid") +BLKDEV_CMD = subp.which("blockdev") +WIPEFS_CMD = subp.which("wipefs") LANG_C_ENV = {'LANG': 'C'} @@ -248,7 +249,7 @@ def enumerate_disk(device, nodeps=False): info = None try: - info, _err = util.subp(lsblk_cmd) + info, _err = subp.subp(lsblk_cmd) except Exception as e: raise Exception("Failed during disk check for %s\n%s" % (device, e)) @@ -310,7 +311,7 @@ def check_fs(device): blkid_cmd = [BLKID_CMD, '-c', '/dev/null', device] try: - out, _err = util.subp(blkid_cmd, rcs=[0, 2]) + out, _err = subp.subp(blkid_cmd, rcs=[0, 2]) except Exception as e: raise Exception("Failed during disk check for %s\n%s" % (device, e)) @@ -433,8 +434,8 @@ def get_dyn_func(*args): def get_hdd_size(device): try: - size_in_bytes, _ = util.subp([BLKDEV_CMD, '--getsize64', device]) - sector_size, _ = util.subp([BLKDEV_CMD, '--getss', device]) + size_in_bytes, _ = subp.subp([BLKDEV_CMD, '--getsize64', device]) + sector_size, _ = subp.subp([BLKDEV_CMD, '--getss', device]) except Exception as e: raise Exception("Failed to get %s size\n%s" % (device, e)) @@ -452,7 +453,7 @@ def check_partition_mbr_layout(device, layout): read_parttbl(device) prt_cmd = [SFDISK_CMD, "-l", device] try: - out, _err = util.subp(prt_cmd, data="%s\n" % layout) + out, _err = subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: raise Exception("Error running partition command on %s\n%s" % ( device, e)) @@ -482,7 +483,7 @@ def check_partition_mbr_layout(device, layout): def check_partition_gpt_layout(device, layout): prt_cmd = [SGDISK_CMD, '-p', device] try: - out, _err = util.subp(prt_cmd, update_env=LANG_C_ENV) + out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV) except Exception as e: raise Exception("Error running partition command on %s\n%s" % ( device, e)) @@ -655,7 +656,7 @@ def purge_disk(device): wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']] try: LOG.info("Purging filesystem on /dev/%s", d['name']) - util.subp(wipefs_cmd) + subp.subp(wipefs_cmd) except Exception: raise Exception("Failed FS purge of /dev/%s" % d['name']) @@ -682,7 +683,7 @@ def read_parttbl(device): blkdev_cmd = [BLKDEV_CMD, '--rereadpt', device] util.udevadm_settle() try: - util.subp(blkdev_cmd) + subp.subp(blkdev_cmd) except Exception as e: util.logexc(LOG, "Failed reading the partition table %s" % e) @@ -697,7 +698,7 @@ def exec_mkpart_mbr(device, layout): # Create the partitions prt_cmd = [SFDISK_CMD, "--Linux", "--unit=S", "--force", device] try: - util.subp(prt_cmd, data="%s\n" % layout) + subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: raise Exception("Failed to partition device %s\n%s" % (device, e)) @@ -706,16 +707,16 @@ def exec_mkpart_mbr(device, layout): def exec_mkpart_gpt(device, layout): try: - util.subp([SGDISK_CMD, '-Z', device]) + subp.subp([SGDISK_CMD, '-Z', device]) for index, (partition_type, (start, end)) in enumerate(layout): index += 1 - util.subp([SGDISK_CMD, + subp.subp([SGDISK_CMD, '-n', '{}:{}:{}'.format(index, start, end), device]) if partition_type is not None: # convert to a 4 char (or more) string right padded with 0 # 82 -> 8200. 'Linux' -> 'Linux' pinput = str(partition_type).ljust(4, "0") - util.subp( + subp.subp( [SGDISK_CMD, '-t', '{}:{}'.format(index, pinput), device]) except Exception: LOG.warning("Failed to partition device %s", device) @@ -967,9 +968,9 @@ def mkfs(fs_cfg): fs_cmd) else: # Find the mkfs command - mkfs_cmd = util.which("mkfs.%s" % fs_type) + mkfs_cmd = subp.which("mkfs.%s" % fs_type) if not mkfs_cmd: - mkfs_cmd = util.which("mk%s" % fs_type) + mkfs_cmd = subp.which("mk%s" % fs_type) if not mkfs_cmd: LOG.warning("Cannot create fstype '%s'. No mkfs.%s command", @@ -994,7 +995,7 @@ def mkfs(fs_cfg): LOG.debug("Creating file system %s on %s", label, device) LOG.debug(" Using cmd: %s", str(fs_cmd)) try: - util.subp(fs_cmd, shell=shell) + subp.subp(fs_cmd, shell=shell) except Exception as e: raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index b342e04d..b1d99f97 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -25,7 +25,7 @@ import os from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS -from cloudinit import util +from cloudinit import subp frequency = PER_ALWAYS @@ -43,9 +43,9 @@ def is_upstart_system(): del myenv['UPSTART_SESSION'] check_cmd = ['initctl', 'version'] try: - (out, _err) = util.subp(check_cmd, env=myenv) + (out, _err) = subp.subp(check_cmd, env=myenv) return 'upstart' in out - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: LOG.debug("'%s' returned '%s', not using upstart", ' '.join(check_cmd), e.exit_code) return False @@ -66,7 +66,7 @@ def handle(name, _cfg, cloud, log, args): for n in event_names: cmd = ['initctl', 'emit', str(n), 'CLOUD_CFG=%s' % cfgpath] try: - util.subp(cmd) + subp.subp(cmd) except Exception as e: # TODO(harlowja), use log exception from utils?? log.warning("Emission of upstart event %s failed due to: %s", n, e) diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py index 0a135bbe..77984bca 100644 --- a/cloudinit/config/cc_fan.py +++ b/cloudinit/config/cc_fan.py @@ -39,6 +39,7 @@ If cloud-init sees a ``fan`` entry in cloud-config it will: from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -62,8 +63,8 @@ def stop_update_start(service, config_file, content, systemd=False): def run(cmd, msg): try: - return util.subp(cmd, capture=True) - except util.ProcessExecutionError as e: + return subp.subp(cmd, capture=True) + except subp.ProcessExecutionError as e: LOG.warning("failed: %s (%s): %s", service, cmd, e) return False @@ -94,7 +95,7 @@ def handle(name, cfg, cloud, log, args): util.write_file(mycfg.get('config_path'), mycfg.get('config'), omode="w") distro = cloud.distro - if not util.which('fanctl'): + if not subp.which('fanctl'): distro.install_packages(['ubuntu-fan']) stop_update_start( diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 1b512a06..c5d93f81 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -70,6 +70,7 @@ import stat from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS +from cloudinit import subp from cloudinit import util frequency = PER_ALWAYS @@ -131,19 +132,19 @@ class ResizeGrowPart(object): myenv['LANG'] = 'C' try: - (out, _err) = util.subp(["growpart", "--help"], env=myenv) + (out, _err) = subp.subp(["growpart", "--help"], env=myenv) if re.search(r"--update\s+", out): return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass return False def resize(self, diskdev, partnum, partdev): before = get_size(partdev) try: - util.subp(["growpart", '--dry-run', diskdev, partnum]) - except util.ProcessExecutionError as e: + subp.subp(["growpart", '--dry-run', diskdev, partnum]) + except subp.ProcessExecutionError as e: if e.exit_code != 1: util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)", diskdev, partnum) @@ -151,8 +152,8 @@ class ResizeGrowPart(object): return (before, before) try: - util.subp(["growpart", diskdev, partnum]) - except util.ProcessExecutionError as e: + subp.subp(["growpart", diskdev, partnum]) + except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: growpart %s %s", diskdev, partnum) raise ResizeFailedException(e) @@ -165,11 +166,11 @@ class ResizeGpart(object): myenv['LANG'] = 'C' try: - (_out, err) = util.subp(["gpart", "help"], env=myenv, rcs=[0, 1]) + (_out, err) = subp.subp(["gpart", "help"], env=myenv, rcs=[0, 1]) if re.search(r"gpart recover ", err): return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass return False @@ -182,16 +183,16 @@ class ResizeGpart(object): be recovered. """ try: - util.subp(["gpart", "recover", diskdev]) - except util.ProcessExecutionError as e: + subp.subp(["gpart", "recover", diskdev]) + except subp.ProcessExecutionError as e: if e.exit_code != 0: util.logexc(LOG, "Failed: gpart recover %s", diskdev) raise ResizeFailedException(e) before = get_size(partdev) try: - util.subp(["gpart", "resize", "-i", partnum, diskdev]) - except util.ProcessExecutionError as e: + subp.subp(["gpart", "resize", "-i", partnum, diskdev]) + except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev) raise ResizeFailedException(e) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 7888464e..eb03c664 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -43,8 +43,9 @@ seeded with empty values, and install_devices_empty is set to true. import os +from cloudinit import subp from cloudinit import util -from cloudinit.util import ProcessExecutionError +from cloudinit.subp import ProcessExecutionError distros = ['ubuntu', 'debian'] @@ -59,7 +60,7 @@ def fetch_idevs(log): try: # get the root disk where the /boot directory resides. - disk = util.subp(['grub-probe', '-t', 'disk', '/boot'], + disk = subp.subp(['grub-probe', '-t', 'disk', '/boot'], capture=True)[0].strip() except ProcessExecutionError as e: # grub-common may not be installed, especially on containers @@ -84,7 +85,7 @@ def fetch_idevs(log): try: # check if disk exists and use udevadm to fetch symlinks - devices = util.subp( + devices = subp.subp( ['udevadm', 'info', '--root', '--query=symlink', disk], capture=True )[0].strip().split() @@ -135,7 +136,7 @@ def handle(name, cfg, _cloud, log, _args): (idevs, idevs_empty)) try: - util.subp(['debconf-set-selections'], dconf_sel) + subp.subp(['debconf-set-selections'], dconf_sel) except Exception: util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg") diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index 3d2ded3d..0f2be52b 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -33,6 +33,7 @@ key can be used. By default ``ssh-dss`` keys are not written to console. import os from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util frequency = PER_INSTANCE @@ -64,7 +65,7 @@ def handle(name, cfg, cloud, log, _args): try: cmd = [helper_path, ','.join(fp_blacklist), ','.join(key_blacklist)] - (stdout, _stderr) = util.subp(cmd) + (stdout, _stderr) = subp.subp(cmd) util.multi_log("%s\n" % (stdout.strip()), stderr=False, console=True) except Exception: diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index a9c04d86..299c4d01 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -61,6 +61,7 @@ from io import BytesIO from configobj import ConfigObj from cloudinit import type_utils +from cloudinit import subp from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -116,7 +117,7 @@ def handle(_name, cfg, cloud, log, _args): log.debug("Wrote landscape config file to %s", LSC_CLIENT_CFG_FILE) util.write_file(LS_DEFAULT_FILE, "RUN=1\n") - util.subp(["service", "landscape-client", "restart"]) + subp.subp(["service", "landscape-client", "restart"]) def merge_together(objs): diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 151a9844..7129c9c6 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -48,6 +48,7 @@ lxd-bridge will be configured accordingly. """ from cloudinit import log as logging +from cloudinit import subp from cloudinit import util import os @@ -85,16 +86,16 @@ def handle(name, cfg, cloud, log, args): # Install the needed packages packages = [] - if not util.which("lxd"): + if not subp.which("lxd"): packages.append('lxd') - if init_cfg.get("storage_backend") == "zfs" and not util.which('zfs'): + if init_cfg.get("storage_backend") == "zfs" and not subp.which('zfs'): packages.append('zfsutils-linux') if len(packages): try: cloud.distro.install_packages(packages) - except util.ProcessExecutionError as exc: + except subp.ProcessExecutionError as exc: log.warning("failed to install packages %s: %s", packages, exc) return @@ -104,20 +105,20 @@ def handle(name, cfg, cloud, log, args): 'network_address', 'network_port', 'storage_backend', 'storage_create_device', 'storage_create_loop', 'storage_pool', 'trust_password') - util.subp(['lxd', 'waitready', '--timeout=300']) + subp.subp(['lxd', 'waitready', '--timeout=300']) cmd = ['lxd', 'init', '--auto'] for k in init_keys: if init_cfg.get(k): cmd.extend(["--%s=%s" % (k.replace('_', '-'), str(init_cfg[k]))]) - util.subp(cmd) + subp.subp(cmd) # Set up lxd-bridge if bridge config is given dconf_comm = "debconf-communicate" if bridge_cfg: net_name = bridge_cfg.get("name", _DEFAULT_NETWORK_NAME) if os.path.exists("/etc/default/lxd-bridge") \ - and util.which(dconf_comm): + and subp.which(dconf_comm): # Bridge configured through packaging debconf = bridge_to_debconf(bridge_cfg) @@ -127,7 +128,7 @@ def handle(name, cfg, cloud, log, args): log.debug("Setting lxd debconf via " + dconf_comm) data = "\n".join(["set %s %s" % (k, v) for k, v in debconf.items()]) + "\n" - util.subp(['debconf-communicate'], data) + subp.subp(['debconf-communicate'], data) except Exception: util.logexc(log, "Failed to run '%s' for lxd with" % dconf_comm) @@ -137,7 +138,7 @@ def handle(name, cfg, cloud, log, args): # Run reconfigure log.debug("Running dpkg-reconfigure for lxd") - util.subp(['dpkg-reconfigure', 'lxd', + subp.subp(['dpkg-reconfigure', 'lxd', '--frontend=noninteractive']) else: # Built-in LXD bridge support @@ -264,7 +265,7 @@ def _lxc(cmd): env = {'LC_ALL': 'C', 'HOME': os.environ.get('HOME', '/root'), 'USER': os.environ.get('USER', 'root')} - util.subp(['lxc'] + list(cmd) + ["--force-local"], update_env=env) + subp.subp(['lxc'] + list(cmd) + ["--force-local"], update_env=env) def maybe_cleanup_default(net_name, did_init, create, attach, @@ -286,7 +287,7 @@ def maybe_cleanup_default(net_name, did_init, create, attach, try: _lxc(["network", "delete", net_name]) LOG.debug(msg, net_name, succeeded) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code != 1: raise e LOG.debug(msg, net_name, fail_assume_enoent) @@ -296,7 +297,7 @@ def maybe_cleanup_default(net_name, did_init, create, attach, try: _lxc(["profile", "device", "remove", profile, nic_name]) LOG.debug(msg, nic_name, profile, succeeded) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code != 1: raise e LOG.debug(msg, nic_name, profile, fail_assume_enoent) diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py index 351183f1..41ea4fc9 100644 --- a/cloudinit/config/cc_mcollective.py +++ b/cloudinit/config/cc_mcollective.py @@ -56,6 +56,7 @@ import io from configobj import ConfigObj from cloudinit import log as logging +from cloudinit import subp from cloudinit import util PUBCERT_FILE = "/etc/mcollective/ssl/server-public.pem" @@ -140,6 +141,6 @@ def handle(name, cfg, cloud, log, _args): configure(config=mcollective_cfg['conf']) # restart mcollective to handle updated config - util.subp(['service', 'mcollective', 'restart'], capture=False) + subp.subp(['service', 'mcollective', 'restart'], capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 85a89cd1..e57d1b1f 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -69,6 +69,7 @@ import os.path import re from cloudinit import type_utils +from cloudinit import subp from cloudinit import util # Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0 @@ -252,8 +253,8 @@ def create_swapfile(fname: str, size: str) -> None: 'count=%s' % size] try: - util.subp(cmd, capture=True) - except util.ProcessExecutionError as e: + subp.subp(cmd, capture=True) + except subp.ProcessExecutionError as e: LOG.warning(errmsg, fname, size, method, e) util.del_file(fname) @@ -267,15 +268,15 @@ def create_swapfile(fname: str, size: str) -> None: else: try: create_swap(fname, size, "fallocate") - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: LOG.warning(errmsg, fname, size, "dd", e) LOG.warning("Will attempt with dd.") create_swap(fname, size, "dd") util.chmod(fname, 0o600) try: - util.subp(['mkswap', fname]) - except util.ProcessExecutionError: + subp.subp(['mkswap', fname]) + except subp.ProcessExecutionError: util.del_file(fname) raise @@ -538,9 +539,9 @@ def handle(_name, cfg, cloud, log, _args): for cmd in activate_cmds: fmt = "Activate mounts: %s:" + ' '.join(cmd) try: - util.subp(cmd) + subp.subp(cmd) log.debug(fmt, "PASS") - except util.ProcessExecutionError: + except subp.ProcessExecutionError: log.warning(fmt, "FAIL") util.logexc(log, fmt, "FAIL") diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 3b2c2020..7d3f73ff 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -14,6 +14,7 @@ from cloudinit import log as logging from cloudinit import temp_utils from cloudinit import templater from cloudinit import type_utils +from cloudinit import subp from cloudinit import util from cloudinit.config.schema import get_schema_doc, validate_cloudconfig_schema from cloudinit.settings import PER_INSTANCE @@ -307,7 +308,7 @@ def select_ntp_client(ntp_client, distro): if distro_ntp_client == "auto": for client in distro.preferred_ntp_clients: cfg = distro_cfg.get(client) - if util.which(cfg.get('check_exe')): + if subp.which(cfg.get('check_exe')): LOG.debug('Selected NTP client "%s", already installed', client) clientcfg = cfg @@ -336,7 +337,7 @@ def install_ntp_client(install_func, packages=None, check_exe="ntpd"): @param check_exe: string. The name of a binary that indicates the package the specified package is already installed. """ - if util.which(check_exe): + if subp.which(check_exe): return if packages is None: packages = ['ntp'] @@ -431,7 +432,7 @@ def reload_ntp(service, systemd=False): cmd = ['systemctl', 'reload-or-restart', service] else: cmd = ['service', service, 'restart'] - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def supplemental_schema_validation(ntp_config): @@ -543,7 +544,7 @@ def handle(name, cfg, cloud, log, _args): try: reload_ntp(ntp_client_config['service_name'], systemd=cloud.distro.uses_systemd()) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: LOG.exception("Failed to reload/start ntp service: %s", e) raise diff --git a/cloudinit/config/cc_package_update_upgrade_install.py b/cloudinit/config/cc_package_update_upgrade_install.py index 86afffef..036baf85 100644 --- a/cloudinit/config/cc_package_update_upgrade_install.py +++ b/cloudinit/config/cc_package_update_upgrade_install.py @@ -43,6 +43,7 @@ import os import time from cloudinit import log as logging +from cloudinit import subp from cloudinit import util REBOOT_FILE = "/var/run/reboot-required" @@ -57,7 +58,7 @@ def _multi_cfg_bool_get(cfg, *keys): def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): - util.subp(REBOOT_CMD) + subp.subp(REBOOT_CMD) start = time.time() wait_time = initial_sleep for _i in range(0, wait_attempts): diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 3e81a3c7..41ffb46c 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -56,6 +56,7 @@ import subprocess import time from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util frequency = PER_INSTANCE @@ -71,7 +72,7 @@ def givecmdline(pid): # PID COMM ARGS # 1 init /bin/init -- if util.is_FreeBSD(): - (output, _err) = util.subp(['procstat', '-c', str(pid)]) + (output, _err) = subp.subp(['procstat', '-c', str(pid)]) line = output.splitlines()[1] m = re.search(r'\d+ (\w|\.|-)+\s+(/\w.+)', line) return m.group(2) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index c01f5b8f..635c73bc 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -83,6 +83,7 @@ import yaml from io import StringIO from cloudinit import helpers +from cloudinit import subp from cloudinit import util PUPPET_CONF_PATH = '/etc/puppet/puppet.conf' @@ -105,14 +106,14 @@ class PuppetConstants(object): def _autostart_puppet(log): # Set puppet to automatically start if os.path.exists('/etc/default/puppet'): - util.subp(['sed', '-i', + subp.subp(['sed', '-i', '-e', 's/^START=.*/START=yes/', '/etc/default/puppet'], capture=False) elif os.path.exists('/bin/systemctl'): - util.subp(['/bin/systemctl', 'enable', 'puppet.service'], + subp.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False) elif os.path.exists('/sbin/chkconfig'): - util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) + subp.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) else: log.warning(("Sorry we do not know how to enable" " puppet services on this system")) @@ -203,6 +204,6 @@ def handle(name, cfg, cloud, log, _args): _autostart_puppet(log) # Start puppetd - util.subp(['service', 'puppet', 'start'], capture=False) + subp.subp(['service', 'puppet', 'start'], capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 01dfc125..8de4db30 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -19,6 +19,7 @@ from textwrap import dedent from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_ALWAYS +from cloudinit import subp from cloudinit import util NOBLOCK = "noblock" @@ -88,11 +89,11 @@ def _resize_zfs(mount_point, devpth): def _get_dumpfs_output(mount_point): - return util.subp(['dumpfs', '-m', mount_point])[0] + return subp.subp(['dumpfs', '-m', mount_point])[0] def _get_gpart_output(part): - return util.subp(['gpart', 'show', part])[0] + return subp.subp(['gpart', 'show', part])[0] def _can_skip_resize_ufs(mount_point, devpth): @@ -306,8 +307,8 @@ def handle(name, cfg, _cloud, log, args): def do_resize(resize_cmd, log): try: - util.subp(resize_cmd) - except util.ProcessExecutionError: + subp.subp(resize_cmd) + except subp.ProcessExecutionError: util.logexc(log, "Failed to resize filesystem (cmd=%s)", resize_cmd) raise # TODO(harlowja): Should we add a fsck check after this to make diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 28c79b83..28d62e9d 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -39,6 +39,7 @@ Subscription`` example config. """ from cloudinit import log as logging +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -173,7 +174,7 @@ class SubscriptionManager(object): try: _sub_man_cli(cmd) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: return False return True @@ -200,7 +201,7 @@ class SubscriptionManager(object): try: return_out = _sub_man_cli(cmd, logstring_val=True)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.stdout == "": self.log_warn("Registration failed due " "to: {0}".format(e.stderr)) @@ -223,7 +224,7 @@ class SubscriptionManager(object): # Attempting to register the system only try: return_out = _sub_man_cli(cmd, logstring_val=True)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.stdout == "": self.log_warn("Registration failed due " "to: {0}".format(e.stderr)) @@ -246,7 +247,7 @@ class SubscriptionManager(object): try: return_out = _sub_man_cli(cmd)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.stdout.rstrip() != '': for line in e.stdout.split("\n"): if line != '': @@ -264,7 +265,7 @@ class SubscriptionManager(object): cmd = ['attach', '--auto'] try: return_out = _sub_man_cli(cmd)[0] - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: self.log_warn("Auto-attach failed with: {0}".format(e)) return False for line in return_out.split("\n"): @@ -341,7 +342,7 @@ class SubscriptionManager(object): "system: %s", (", ".join(pool_list)) .replace('--pool=', '')) return True - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: self.log_warn("Unable to attach pool {0} " "due to {1}".format(pool, e)) return False @@ -414,7 +415,7 @@ class SubscriptionManager(object): try: _sub_man_cli(cmd) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: self.log_warn("Unable to alter repos due to {0}".format(e)) return False @@ -432,11 +433,11 @@ class SubscriptionManager(object): def _sub_man_cli(cmd, logstring_val=False): ''' - Uses the prefered cloud-init subprocess def of util.subp + Uses the prefered cloud-init subprocess def of subp.subp and runs subscription-manager. Breaking this to a separate function for later use in mocking and unittests ''' - return util.subp(['subscription-manager'] + cmd, + return subp.subp(['subscription-manager'] + cmd, logstring=logstring_val) diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 5df0137d..1354885a 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -182,6 +182,7 @@ import os import re from cloudinit import log as logging +from cloudinit import subp from cloudinit import util DEF_FILENAME = "20-cloud-config.conf" @@ -215,7 +216,7 @@ def reload_syslog(command=DEF_RELOAD, systemd=False): cmd = ['service', service, 'restart'] else: cmd = command - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def load_config(cfg): @@ -429,7 +430,7 @@ def handle(name, cfg, cloud, log, _args): restarted = reload_syslog( command=mycfg[KEYNAME_RELOAD], systemd=cloud.distro.uses_systemd()), - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: restarted = False log.warning("Failed to reload syslog", e) diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 5dd8de37..b61876aa 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -45,7 +45,7 @@ specify them with ``pkg_name``, ``service_name`` and ``config_dir``. import os -from cloudinit import safeyaml, util +from cloudinit import safeyaml, subp, util from cloudinit.distros import rhel_util @@ -130,6 +130,6 @@ def handle(name, cfg, cloud, log, _args): # restart salt-minion. 'service' will start even if not started. if it # was started, it needs to be restarted for config change. - util.subp(['service', const.srv_name, 'restart'], capture=False) + subp.subp(['service', const.srv_name, 'restart'], capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index b65f3ed9..4fb9b44e 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -65,6 +65,7 @@ from io import BytesIO from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util frequency = PER_INSTANCE @@ -92,14 +93,14 @@ def handle_random_seed_command(command, required, env=None): return cmd = command[0] - if not util.which(cmd): + if not subp.which(cmd): if required: raise ValueError( "command '{cmd}' not found but required=true".format(cmd=cmd)) else: LOG.debug("command '%s' not found for seed_command", cmd) return - util.subp(command, env=env, capture=False) + subp.subp(command, env=env, capture=False) def handle(name, cfg, cloud, log, _args): diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 7b7aa885..d6b5682d 100755 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -83,6 +83,7 @@ import sys from cloudinit.distros import ug_util from cloudinit import log as logging from cloudinit.ssh_util import update_ssh_config +from cloudinit import subp from cloudinit import util from string import ascii_letters, digits @@ -128,7 +129,7 @@ def handle_ssh_pwauth(pw_auth, service_cmd=None, service_name="ssh"): cmd = list(service_cmd) + ["restart", service_name] else: cmd = list(service_cmd) + [service_name, "restart"] - util.subp(cmd) + subp.subp(cmd) LOG.debug("Restarted the SSH daemon.") @@ -247,6 +248,6 @@ def chpasswd(distro, plist_in, hashed=False): distro.set_passwd(u, p, hashed=hashed) else: cmd = ['chpasswd'] + (['-e'] if hashed else []) - util.subp(cmd, plist_in) + subp.subp(cmd, plist_in) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py index 8178562e..20ed7d2f 100644 --- a/cloudinit/config/cc_snap.py +++ b/cloudinit/config/cc_snap.py @@ -12,6 +12,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_INSTANCE from cloudinit.subp import prepend_base_command +from cloudinit import subp from cloudinit import util @@ -175,7 +176,7 @@ def add_assertions(assertions): LOG.debug('Snap acking: %s', asrt.split('\n')[0:2]) util.write_file(ASSERTIONS_FILE, combined.encode('utf-8')) - util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) + subp.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) def run_commands(commands): @@ -204,8 +205,8 @@ def run_commands(commands): for command in fixed_snap_commands: shell = isinstance(command, str) try: - util.subp(command, shell=shell, status_cb=sys.stderr.write) - except util.ProcessExecutionError as e: + subp.subp(command, shell=shell, status_cb=sys.stderr.write) + except subp.ProcessExecutionError as e: cmd_failures.append(str(e)) if cmd_failures: msg = 'Failures running snap commands:\n{cmd_failures}'.format( diff --git a/cloudinit/config/cc_spacewalk.py b/cloudinit/config/cc_spacewalk.py index 1020e944..95083607 100644 --- a/cloudinit/config/cc_spacewalk.py +++ b/cloudinit/config/cc_spacewalk.py @@ -27,7 +27,7 @@ For more information about spacewalk see: https://fedorahosted.org/spacewalk/ activation_key: """ -from cloudinit import util +from cloudinit import subp distros = ['redhat', 'fedora'] @@ -41,9 +41,9 @@ def is_registered(): # assume we aren't registered; which is sorta ghetto... already_registered = False try: - util.subp(['rhn-profile-sync', '--verbose'], capture=False) + subp.subp(['rhn-profile-sync', '--verbose'], capture=False) already_registered = True - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code != 1: raise return already_registered @@ -65,7 +65,7 @@ def do_register(server, profile_name, cmd.extend(['--sslCACert', str(ca_cert_path)]) if activation_key: cmd.extend(['--activationkey', str(activation_key)]) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def handle(name, cfg, cloud, log, _args): diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 163cce99..228e5e0d 100755 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -116,6 +116,7 @@ import sys from cloudinit.distros import ug_util from cloudinit import ssh_util +from cloudinit import subp from cloudinit import util @@ -164,7 +165,7 @@ def handle(_name, cfg, cloud, log, _args): try: # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) except Exception: util.logexc(log, "Failed generated a key for %s from %s", @@ -186,9 +187,9 @@ def handle(_name, cfg, cloud, log, _args): # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): try: - out, err = util.subp(cmd, capture=True, env=lang_c) + out, err = subp.subp(cmd, capture=True, env=lang_c) sys.stdout.write(util.decode_binary(out)) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: err = util.decode_binary(e.stderr).lower() if (e.exit_code == 1 and err.lower().startswith("unknown key")): diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 63f87298..856e5a9e 100755 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -31,6 +31,7 @@ either ``lp:`` for launchpad or ``gh:`` for github to the username. """ from cloudinit.distros import ug_util +from cloudinit import subp from cloudinit import util import pwd @@ -101,8 +102,8 @@ def import_ssh_ids(ids, user, log): log.debug("Importing SSH ids for user %s.", user) try: - util.subp(cmd, capture=False) - except util.ProcessExecutionError as exc: + subp.subp(cmd, capture=False) + except subp.ProcessExecutionError as exc: util.logexc(log, "Failed to run command to import %s SSH ids", user) raise exc diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py index 8b6d2a1a..35ded5db 100644 --- a/cloudinit/config/cc_ubuntu_advantage.py +++ b/cloudinit/config/cc_ubuntu_advantage.py @@ -8,6 +8,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import util @@ -109,8 +110,8 @@ def configure_ua(token=None, enable=None): attach_cmd = ['ua', 'attach', token] LOG.debug('Attaching to Ubuntu Advantage. %s', ' '.join(attach_cmd)) try: - util.subp(attach_cmd) - except util.ProcessExecutionError as e: + subp.subp(attach_cmd) + except subp.ProcessExecutionError as e: msg = 'Failure attaching Ubuntu Advantage:\n{error}'.format( error=str(e)) util.logexc(LOG, msg) @@ -119,8 +120,8 @@ def configure_ua(token=None, enable=None): for service in enable: try: cmd = ['ua', 'enable', service] - util.subp(cmd, capture=True) - except util.ProcessExecutionError as e: + subp.subp(cmd, capture=True) + except subp.ProcessExecutionError as e: enable_errors.append((service, e)) if enable_errors: for service, error in enable_errors: @@ -135,7 +136,7 @@ def configure_ua(token=None, enable=None): def maybe_install_ua_tools(cloud): """Install ubuntu-advantage-tools if not present.""" - if util.which('ua'): + if subp.which('ua'): return try: cloud.distro.update_package_sources() diff --git a/cloudinit/config/cc_ubuntu_drivers.py b/cloudinit/config/cc_ubuntu_drivers.py index 297451d6..2d1d2b32 100644 --- a/cloudinit/config/cc_ubuntu_drivers.py +++ b/cloudinit/config/cc_ubuntu_drivers.py @@ -9,6 +9,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import subp from cloudinit import temp_utils from cloudinit import type_utils from cloudinit import util @@ -108,7 +109,7 @@ def install_drivers(cfg, pkg_install_func): LOG.debug("Not installing NVIDIA drivers. %s=%s", cfgpath, nv_acc) return - if not util.which('ubuntu-drivers'): + if not subp.which('ubuntu-drivers'): LOG.debug("'ubuntu-drivers' command not available. " "Installing ubuntu-drivers-common") pkg_install_func(['ubuntu-drivers-common']) @@ -131,7 +132,7 @@ def install_drivers(cfg, pkg_install_func): debconf_script, util.encode_text(NVIDIA_DRIVER_LATELINK_DEBCONF_SCRIPT), mode=0o755) - util.subp([debconf_script, debconf_file]) + subp.subp([debconf_script, debconf_file]) except Exception as e: util.logexc( LOG, "Failed to register NVIDIA debconf template: %s", str(e)) @@ -141,8 +142,8 @@ def install_drivers(cfg, pkg_install_func): util.del_dir(tdir) try: - util.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg]) - except util.ProcessExecutionError as exc: + subp.subp(['ubuntu-drivers', 'install', '--gpgpu', driver_arg]) + except subp.ProcessExecutionError as exc: if OLD_UBUNTU_DRIVERS_STDERR_NEEDLE in exc.stderr: LOG.warning('the available version of ubuntu-drivers is' ' too old to perform requested driver installation') diff --git a/cloudinit/config/tests/test_disable_ec2_metadata.py b/cloudinit/config/tests/test_disable_ec2_metadata.py index 823917c7..b00f2083 100644 --- a/cloudinit/config/tests/test_disable_ec2_metadata.py +++ b/cloudinit/config/tests/test_disable_ec2_metadata.py @@ -15,8 +15,8 @@ DISABLE_CFG = {'disable_ec2_metadata': 'true'} class TestEC2MetadataRoute(CiTestCase): - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp') def test_disable_ifconfig(self, m_subp, m_which): """Set the route if ifconfig command is available""" m_which.side_effect = lambda x: x if x == 'ifconfig' else None @@ -25,8 +25,8 @@ class TestEC2MetadataRoute(CiTestCase): ['route', 'add', '-host', '169.254.169.254', 'reject'], capture=False) - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp') def test_disable_ip(self, m_subp, m_which): """Set the route if ip command is available""" m_which.side_effect = lambda x: x if x == 'ip' else None @@ -35,8 +35,8 @@ class TestEC2MetadataRoute(CiTestCase): ['ip', 'route', 'add', 'prohibit', '169.254.169.254'], capture=False) - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.which') - @mock.patch('cloudinit.config.cc_disable_ec2_metadata.util.subp') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.which') + @mock.patch('cloudinit.config.cc_disable_ec2_metadata.subp.subp') def test_disable_no_tool(self, m_subp, m_which): """Log error when neither route nor ip commands are available""" m_which.return_value = None # Find neither ifconfig nor ip diff --git a/cloudinit/config/tests/test_grub_dpkg.py b/cloudinit/config/tests/test_grub_dpkg.py index 01efa330..99c05bb5 100644 --- a/cloudinit/config/tests/test_grub_dpkg.py +++ b/cloudinit/config/tests/test_grub_dpkg.py @@ -4,7 +4,7 @@ import pytest from unittest import mock from logging import Logger -from cloudinit.util import ProcessExecutionError +from cloudinit.subp import ProcessExecutionError from cloudinit.config.cc_grub_dpkg import fetch_idevs, handle @@ -79,7 +79,7 @@ class TestFetchIdevs: ) @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc") @mock.patch("cloudinit.config.cc_grub_dpkg.os.path.exists") - @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp") + @mock.patch("cloudinit.config.cc_grub_dpkg.subp.subp") def test_fetch_idevs(self, m_subp, m_exists, m_logexc, grub_output, path_exists, expected_log_call, udevadm_output, expected_idevs): @@ -158,7 +158,7 @@ class TestHandle: @mock.patch("cloudinit.config.cc_grub_dpkg.fetch_idevs") @mock.patch("cloudinit.config.cc_grub_dpkg.util.get_cfg_option_str") @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc") - @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp") + @mock.patch("cloudinit.config.cc_grub_dpkg.subp.subp") def test_handle(self, m_subp, m_logexc, m_get_cfg_str, m_fetch_idevs, cfg_idevs, cfg_idevs_empty, fetch_idevs_output, expected_log_output): diff --git a/cloudinit/config/tests/test_mounts.py b/cloudinit/config/tests/test_mounts.py index 80b54d0f..764a33e3 100644 --- a/cloudinit/config/tests/test_mounts.py +++ b/cloudinit/config/tests/test_mounts.py @@ -13,12 +13,12 @@ class TestCreateSwapfile: @pytest.mark.parametrize('fstype', ('xfs', 'btrfs', 'ext4', 'other')) @mock.patch(M_PATH + 'util.get_mount_info') - @mock.patch(M_PATH + 'util.subp') + @mock.patch(M_PATH + 'subp.subp') def test_happy_path(self, m_subp, m_get_mount_info, fstype, tmpdir): swap_file = tmpdir.join("swap-file") fname = str(swap_file) - # Some of the calls to util.subp should create the swap file; this + # Some of the calls to subp.subp should create the swap file; this # roughly approximates that m_subp.side_effect = lambda *args, **kwargs: swap_file.write('') diff --git a/cloudinit/config/tests/test_set_passwords.py b/cloudinit/config/tests/test_set_passwords.py index 2732bd60..daa1ef51 100644 --- a/cloudinit/config/tests/test_set_passwords.py +++ b/cloudinit/config/tests/test_set_passwords.py @@ -14,7 +14,7 @@ class TestHandleSshPwauth(CiTestCase): with_logs = True - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_unknown_value_logs_warning(self, m_subp): setpass.handle_ssh_pwauth("floo") self.assertIn("Unrecognized value: ssh_pwauth=floo", @@ -22,7 +22,7 @@ class TestHandleSshPwauth(CiTestCase): m_subp.assert_not_called() @mock.patch(MODPATH + "update_ssh_config", return_value=True) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_systemctl_as_service_cmd(self, m_subp, m_update_ssh_config): """If systemctl in service cmd: systemctl restart name.""" setpass.handle_ssh_pwauth( @@ -31,7 +31,7 @@ class TestHandleSshPwauth(CiTestCase): m_subp.call_args) @mock.patch(MODPATH + "update_ssh_config", return_value=True) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_service_as_service_cmd(self, m_subp, m_update_ssh_config): """If systemctl in service cmd: systemctl restart name.""" setpass.handle_ssh_pwauth( @@ -40,7 +40,7 @@ class TestHandleSshPwauth(CiTestCase): m_subp.call_args) @mock.patch(MODPATH + "update_ssh_config", return_value=False) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_not_restarted_if_not_updated(self, m_subp, m_update_ssh_config): """If config is not updated, then no system restart should be done.""" setpass.handle_ssh_pwauth(True) @@ -48,7 +48,7 @@ class TestHandleSshPwauth(CiTestCase): self.assertIn("No need to restart SSH", self.logs.getvalue()) @mock.patch(MODPATH + "update_ssh_config", return_value=True) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_unchanged_does_nothing(self, m_subp, m_update_ssh_config): """If 'unchanged', then no updates to config and no restart.""" setpass.handle_ssh_pwauth( @@ -56,7 +56,7 @@ class TestHandleSshPwauth(CiTestCase): m_update_ssh_config.assert_not_called() m_subp.assert_not_called() - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_valid_change_values(self, m_subp): """If value is a valid changen value, then update should be called.""" upname = MODPATH + "update_ssh_config" @@ -88,7 +88,7 @@ class TestSetPasswordsHandle(CiTestCase): 'ssh_pwauth=None\n', self.logs.getvalue()) - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_handle_on_chpasswd_list_parses_common_hashes(self, m_subp): """handle parses command password hashes.""" cloud = self.tmp_cloud(distro='ubuntu') @@ -98,7 +98,7 @@ class TestSetPasswordsHandle(CiTestCase): 'ubuntu:$6$5hOurLPO$naywm3Ce0UlmZg9gG2Fl9acWCVEoakMMC7dR52q' 'SDexZbrN9z8yHxhUM2b.sxpguSwOlbOQSW/HpXazGGx3oo1'] cfg = {'chpasswd': {'list': valid_hashed_pwds}} - with mock.patch(MODPATH + 'util.subp') as m_subp: + with mock.patch(MODPATH + 'subp.subp') as m_subp: setpass.handle( 'IGNORED', cfg=cfg, cloud=cloud, log=self.logger, args=[]) self.assertIn( @@ -113,7 +113,7 @@ class TestSetPasswordsHandle(CiTestCase): m_subp.call_args_list) @mock.patch(MODPATH + "util.is_BSD") - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_bsd_calls_custom_pw_cmds_to_set_and_expire_passwords( self, m_subp, m_is_bsd): """BSD don't use chpasswd""" @@ -130,7 +130,7 @@ class TestSetPasswordsHandle(CiTestCase): m_subp.call_args_list) @mock.patch(MODPATH + "util.is_BSD") - @mock.patch(MODPATH + "util.subp") + @mock.patch(MODPATH + "subp.subp") def test_handle_on_chpasswd_list_creates_random_passwords(self, m_subp, m_is_bsd): """handle parses command set random passwords.""" @@ -140,7 +140,7 @@ class TestSetPasswordsHandle(CiTestCase): 'root:R', 'ubuntu:RANDOM'] cfg = {'chpasswd': {'expire': 'false', 'list': valid_random_pwds}} - with mock.patch(MODPATH + 'util.subp') as m_subp: + with mock.patch(MODPATH + 'subp.subp') as m_subp: setpass.handle( 'IGNORED', cfg=cfg, cloud=cloud, log=self.logger, args=[]) self.assertIn( diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py index 95270fa0..6d4c014a 100644 --- a/cloudinit/config/tests/test_snap.py +++ b/cloudinit/config/tests/test_snap.py @@ -92,7 +92,7 @@ class TestAddAssertions(CiTestCase): super(TestAddAssertions, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_add_assertions_on_empty_list(self, m_subp): """When provided with an empty list, add_assertions does nothing.""" add_assertions([]) @@ -107,7 +107,7 @@ class TestAddAssertions(CiTestCase): "assertion parameter was not a list or dict: I'm Not Valid", str(context_manager.exception)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_add_assertions_adds_assertions_as_list(self, m_subp): """When provided with a list, add_assertions adds all assertions.""" self.assertEqual( @@ -130,7 +130,7 @@ class TestAddAssertions(CiTestCase): self.assertEqual( util.load_file(compare_file), util.load_file(assert_file)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_add_assertions_adds_assertions_as_dict(self, m_subp): """When provided with a dict, add_assertions adds all assertions.""" self.assertEqual( @@ -168,7 +168,7 @@ class TestRunCommands(CiTestCase): super(TestRunCommands, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_run_commands_on_empty_list(self, m_subp): """When provided with an empty list, run_commands does nothing.""" run_commands([]) @@ -477,7 +477,7 @@ class TestHandle(CiTestCase): self.assertEqual('HI\nMOM\n', util.load_file(outfile)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') def test_handle_adds_assertions(self, m_subp): """Any configured snap assertions are provided to add_assertions.""" assert_file = self.tmp_path('snapd.assertions', dir=self.tmp) @@ -493,7 +493,7 @@ class TestHandle(CiTestCase): self.assertEqual( util.load_file(compare_file), util.load_file(assert_file)) - @mock.patch('cloudinit.config.cc_snap.util.subp') + @mock.patch('cloudinit.config.cc_snap.subp.subp') @skipUnlessJsonSchema() def test_handle_validates_schema(self, m_subp): """Any provided configuration is runs validate_cloudconfig_schema.""" diff --git a/cloudinit/config/tests/test_ubuntu_advantage.py b/cloudinit/config/tests/test_ubuntu_advantage.py index 8c4161ef..db7fb726 100644 --- a/cloudinit/config/tests/test_ubuntu_advantage.py +++ b/cloudinit/config/tests/test_ubuntu_advantage.py @@ -3,7 +3,7 @@ from cloudinit.config.cc_ubuntu_advantage import ( configure_ua, handle, maybe_install_ua_tools, schema) from cloudinit.config.schema import validate_cloudconfig_schema -from cloudinit import util +from cloudinit import subp from cloudinit.tests.helpers import ( CiTestCase, mock, SchemaTestCaseMixin, skipUnlessJsonSchema) @@ -26,10 +26,10 @@ class TestConfigureUA(CiTestCase): super(TestConfigureUA, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_error(self, m_subp): """Errors from ua attach command are raised.""" - m_subp.side_effect = util.ProcessExecutionError( + m_subp.side_effect = subp.ProcessExecutionError( 'Invalid token SomeToken') with self.assertRaises(RuntimeError) as context_manager: configure_ua(token='SomeToken') @@ -39,7 +39,7 @@ class TestConfigureUA(CiTestCase): 'Stdout: Invalid token SomeToken\nStderr: -', str(context_manager.exception)) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_token(self, m_subp): """When token is provided, attach the machine to ua using the token.""" configure_ua(token='SomeToken') @@ -48,7 +48,7 @@ class TestConfigureUA(CiTestCase): 'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n', self.logs.getvalue()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_on_service_error(self, m_subp): """all services should be enabled and then any failures raised""" @@ -56,7 +56,7 @@ class TestConfigureUA(CiTestCase): fail_cmds = [['ua', 'enable', svc] for svc in ['esm', 'cc']] if cmd in fail_cmds and capture: svc = cmd[-1] - raise util.ProcessExecutionError( + raise subp.ProcessExecutionError( 'Invalid {} credentials'.format(svc.upper())) m_subp.side_effect = fake_subp @@ -83,7 +83,7 @@ class TestConfigureUA(CiTestCase): 'Failure enabling Ubuntu Advantage service(s): "esm", "cc"', str(context_manager.exception)) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_empty_services(self, m_subp): """When services is an empty list, do not auto-enable attach.""" configure_ua(token='SomeToken', enable=[]) @@ -92,7 +92,7 @@ class TestConfigureUA(CiTestCase): 'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n', self.logs.getvalue()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_specific_services(self, m_subp): """When services a list, only enable specific services.""" configure_ua(token='SomeToken', enable=['fips']) @@ -105,7 +105,7 @@ class TestConfigureUA(CiTestCase): self.logs.getvalue()) @mock.patch('%s.maybe_install_ua_tools' % MPATH, mock.MagicMock()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_string_services(self, m_subp): """When services a string, treat as singleton list and warn""" configure_ua(token='SomeToken', enable='fips') @@ -119,7 +119,7 @@ class TestConfigureUA(CiTestCase): 'DEBUG: Attaching to Ubuntu Advantage. ua attach SomeToken\n', self.logs.getvalue()) - @mock.patch('%s.util.subp' % MPATH) + @mock.patch('%s.subp.subp' % MPATH) def test_configure_ua_attach_with_weird_services(self, m_subp): """When services not string or list, warn but still attach""" configure_ua(token='SomeToken', enable={'deffo': 'wont work'}) @@ -285,7 +285,7 @@ class TestMaybeInstallUATools(CiTestCase): super(TestMaybeInstallUATools, self).setUp() self.tmp = self.tmp_dir() - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_tools_noop_when_ua_tools_present(self, m_which): """Do nothing if ubuntu-advantage-tools already exists.""" m_which.return_value = '/usr/bin/ua' # already installed @@ -294,7 +294,7 @@ class TestMaybeInstallUATools(CiTestCase): 'Some apt error') maybe_install_ua_tools(cloud=FakeCloud(distro)) # No RuntimeError - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_tools_raises_update_errors(self, m_which): """maybe_install_ua_tools logs and raises apt update errors.""" m_which.return_value = None @@ -306,7 +306,7 @@ class TestMaybeInstallUATools(CiTestCase): self.assertEqual('Some apt error', str(context_manager.exception)) self.assertIn('Package update failed\nTraceback', self.logs.getvalue()) - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_raises_install_errors(self, m_which): """maybe_install_ua_tools logs and raises package install errors.""" m_which.return_value = None @@ -320,7 +320,7 @@ class TestMaybeInstallUATools(CiTestCase): self.assertIn( 'Failed to install ubuntu-advantage-tools\n', self.logs.getvalue()) - @mock.patch('%s.util.which' % MPATH) + @mock.patch('%s.subp.which' % MPATH) def test_maybe_install_ua_tools_happy_path(self, m_which): """maybe_install_ua_tools installs ubuntu-advantage-tools.""" m_which.return_value = None diff --git a/cloudinit/config/tests/test_ubuntu_drivers.py b/cloudinit/config/tests/test_ubuntu_drivers.py index 0aec1265..504ba356 100644 --- a/cloudinit/config/tests/test_ubuntu_drivers.py +++ b/cloudinit/config/tests/test_ubuntu_drivers.py @@ -7,7 +7,7 @@ from cloudinit.tests.helpers import CiTestCase, skipUnlessJsonSchema, mock from cloudinit.config.schema import ( SchemaValidationError, validate_cloudconfig_schema) from cloudinit.config import cc_ubuntu_drivers as drivers -from cloudinit.util import ProcessExecutionError +from cloudinit.subp import ProcessExecutionError MPATH = "cloudinit.config.cc_ubuntu_drivers." M_TMP_PATH = MPATH + "temp_utils.mkdtemp" @@ -53,8 +53,8 @@ class TestUbuntuDrivers(CiTestCase): schema=drivers.schema, strict=True) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=False) def _assert_happy_path_taken( self, config, m_which, m_subp, m_tmp): """Positive path test through handle. Package should be installed.""" @@ -80,8 +80,8 @@ class TestUbuntuDrivers(CiTestCase): self._assert_happy_path_taken(new_config) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp") - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp") + @mock.patch(MPATH + "subp.which", return_value=False) def test_handle_raises_error_if_no_drivers_found( self, m_which, m_subp, m_tmp): """If ubuntu-drivers doesn't install any drivers, raise an error.""" @@ -109,8 +109,8 @@ class TestUbuntuDrivers(CiTestCase): self.assertIn('ubuntu-drivers found no drivers for installation', self.logs.getvalue()) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=False) def _assert_inert_with_config(self, config, m_which, m_subp): """Helper to reduce repetition when testing negative cases""" myCloud = mock.MagicMock() @@ -154,8 +154,8 @@ class TestUbuntuDrivers(CiTestCase): self.assertEqual(0, m_install_drivers.call_count) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=True) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=True) def test_install_drivers_no_install_if_present( self, m_which, m_subp, m_tmp): """If 'ubuntu-drivers' is present, no package install should occur.""" @@ -181,8 +181,8 @@ class TestUbuntuDrivers(CiTestCase): self.assertEqual(0, pkg_install.call_count) @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp") - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp") + @mock.patch(MPATH + "subp.which", return_value=False) def test_install_drivers_handles_old_ubuntu_drivers_gracefully( self, m_which, m_subp, m_tmp): """Older ubuntu-drivers versions should emit message and raise error""" @@ -219,8 +219,8 @@ class TestUbuntuDriversWithVersion(TestUbuntuDrivers): install_gpgpu = ['ubuntu-drivers', 'install', '--gpgpu', 'nvidia:123'] @mock.patch(M_TMP_PATH) - @mock.patch(MPATH + "util.subp", return_value=('', '')) - @mock.patch(MPATH + "util.which", return_value=False) + @mock.patch(MPATH + "subp.subp", return_value=('', '')) + @mock.patch(MPATH + "subp.which", return_value=False) def test_version_none_uses_latest(self, m_which, m_subp, m_tmp): tdir = self.tmp_dir() debconf_file = os.path.join(tdir, 'nvidia.template') diff --git a/cloudinit/conftest.py b/cloudinit/conftest.py index 37cbbcda..251bca59 100644 --- a/cloudinit/conftest.py +++ b/cloudinit/conftest.py @@ -2,32 +2,32 @@ from unittest import mock import pytest -from cloudinit import util +from cloudinit import subp @pytest.yield_fixture(autouse=True) def disable_subp_usage(request): """ - Across all (pytest) tests, ensure that util.subp is not invoked. + Across all (pytest) tests, ensure that subp.subp is not invoked. Note that this can only catch invocations where the util module is imported - and ``util.subp(...)`` is called. ``from cloudinit.util import subp`` + and ``subp.subp(...)`` is called. ``from cloudinit.subp mport subp`` imports happen before the patching here (or the CiTestCase monkey-patching) happens, so are left untouched. - To allow a particular test method or class to use util.subp you can set the + To allow a particular test method or class to use subp.subp you can set the parameter passed to this fixture to False using pytest.mark.parametrize:: @pytest.mark.parametrize("disable_subp_usage", [False], indirect=True) def test_whoami(self): - util.subp(["whoami"]) + subp.subp(["whoami"]) - To instead allow util.subp usage for a specific command, you can set the + To instead allow subp.subp usage for a specific command, you can set the parameter passed to this fixture to that command: @pytest.mark.parametrize("disable_subp_usage", ["bash"], indirect=True) def test_bash(self): - util.subp(["bash"]) + subp.subp(["bash"]) To specify multiple commands, set the parameter to a list (note the double-layered list: we specify a single parameter that is itself a list): @@ -35,8 +35,8 @@ def disable_subp_usage(request): @pytest.mark.parametrize( "disable_subp_usage", ["bash", "whoami"], indirect=True) def test_several_things(self): - util.subp(["bash"]) - util.subp(["whoami"]) + subp.subp(["bash"]) + subp.subp(["whoami"]) This fixture (roughly) mirrors the functionality of CiTestCase.allowed_subp. N.B. While autouse fixtures do affect non-pytest @@ -47,11 +47,11 @@ def disable_subp_usage(request): if should_disable: if not isinstance(should_disable, (list, str)): def side_effect(args, *other_args, **kwargs): - raise AssertionError("Unexpectedly used util.subp") + raise AssertionError("Unexpectedly used subp.subp") else: # Look this up before our patch is in place, so we have access to # the real implementation in side_effect - subp = util.subp + real_subp = subp.subp if isinstance(should_disable, str): should_disable = [should_disable] @@ -60,12 +60,12 @@ def disable_subp_usage(request): cmd = args[0] if cmd not in should_disable: raise AssertionError( - "Unexpectedly used util.subp to call {} (allowed:" + "Unexpectedly used subp.subp to call {} (allowed:" " {})".format(cmd, ",".join(should_disable)) ) - return subp(args, *other_args, **kwargs) + return real_subp(args, *other_args, **kwargs) - with mock.patch('cloudinit.util.subp', autospec=True) as m_subp: + with mock.patch('cloudinit.subp.subp', autospec=True) as m_subp: m_subp.side_effect = side_effect yield else: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 35a10590..016ba64d 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -25,6 +25,7 @@ from cloudinit.net import network_state from cloudinit.net import renderers from cloudinit import ssh_util from cloudinit import type_utils +from cloudinit import subp from cloudinit import util from cloudinit.distros.parsers import hosts @@ -225,8 +226,8 @@ class Distro(metaclass=abc.ABCMeta): LOG.debug("Non-persistently setting the system hostname to %s", hostname) try: - util.subp(['hostname', hostname]) - except util.ProcessExecutionError: + subp.subp(['hostname', hostname]) + except subp.ProcessExecutionError: util.logexc(LOG, "Failed to non-persistently adjust the system " "hostname to %s", hostname) @@ -361,12 +362,12 @@ class Distro(metaclass=abc.ABCMeta): LOG.debug("Attempting to run bring up interface %s using command %s", device_name, cmd) try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False @@ -480,7 +481,7 @@ class Distro(metaclass=abc.ABCMeta): # Run the command LOG.debug("Adding user %s", name) try: - util.subp(useradd_cmd, logstring=log_useradd_cmd) + subp.subp(useradd_cmd, logstring=log_useradd_cmd) except Exception as e: util.logexc(LOG, "Failed to create user %s", name) raise e @@ -500,7 +501,7 @@ class Distro(metaclass=abc.ABCMeta): # Run the command LOG.debug("Adding snap user %s", name) try: - (out, err) = util.subp(create_user_cmd, logstring=create_user_cmd, + (out, err) = subp.subp(create_user_cmd, logstring=create_user_cmd, capture=True) LOG.debug("snap create-user returned: %s:%s", out, err) jobj = util.load_json(out) @@ -582,20 +583,20 @@ class Distro(metaclass=abc.ABCMeta): # passwd must use short '-l' due to SLES11 lacking long form '--lock' lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name]) try: - cmd = next(tool for tool in lock_tools if util.which(tool[0])) + cmd = next(tool for tool in lock_tools if subp.which(tool[0])) except StopIteration: raise RuntimeError(( "Unable to lock user account '%s'. No tools available. " " Tried: %s.") % (name, [c[0] for c in lock_tools])) try: - util.subp(cmd) + subp.subp(cmd) except Exception as e: util.logexc(LOG, 'Failed to disable password for user %s', name) raise e def expire_passwd(self, user): try: - util.subp(['passwd', '--expire', user]) + subp.subp(['passwd', '--expire', user]) except Exception as e: util.logexc(LOG, "Failed to set 'expire' for %s", user) raise e @@ -611,7 +612,7 @@ class Distro(metaclass=abc.ABCMeta): cmd.append('-e') try: - util.subp(cmd, pass_string, logstring="chpasswd for %s" % user) + subp.subp(cmd, pass_string, logstring="chpasswd for %s" % user) except Exception as e: util.logexc(LOG, "Failed to set password for %s", user) raise e @@ -708,7 +709,7 @@ class Distro(metaclass=abc.ABCMeta): LOG.warning("Skipping creation of existing group '%s'", name) else: try: - util.subp(group_add_cmd) + subp.subp(group_add_cmd) LOG.info("Created new group %s", name) except Exception: util.logexc(LOG, "Failed to create group %s", name) @@ -721,7 +722,7 @@ class Distro(metaclass=abc.ABCMeta): "; user does not exist.", member, name) continue - util.subp(['usermod', '-a', '-G', name, member]) + subp.subp(['usermod', '-a', '-G', name, member]) LOG.info("Added user '%s' to group '%s'", member, name) diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 9f89c5f9..038aa9ac 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -8,6 +8,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging from cloudinit import util +from cloudinit import subp from cloudinit.distros import net_util from cloudinit.distros.parsers.hostname import HostnameConf @@ -44,7 +45,7 @@ class Distro(distros.Distro): def apply_locale(self, locale, out_fn=None): if not out_fn: out_fn = self.locale_conf_fn - util.subp(['locale-gen', '-G', locale], capture=False) + subp.subp(['locale-gen', '-G', locale], capture=False) # "" provides trailing newline during join lines = [ util.make_header(), @@ -76,11 +77,11 @@ class Distro(distros.Distro): def _enable_interface(self, device_name): cmd = ['netctl', 'reenable', device_name] try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) def _bring_up_interface(self, device_name): @@ -88,12 +89,12 @@ class Distro(distros.Distro): LOG.debug("Attempting to run bring up interface %s using command %s", device_name, cmd) try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False @@ -158,7 +159,7 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def update_package_sources(self): self._runner.run("update-sources", self.package_command, @@ -173,8 +174,8 @@ def _render_network(entries, target="/", conf_dir="etc/netctl", devs = [] nameservers = [] - resolv_conf = util.target_path(target, resolv_conf) - conf_dir = util.target_path(target, conf_dir) + resolv_conf = subp.target_path(target, resolv_conf) + conf_dir = subp.target_path(target, conf_dir) for (dev, info) in entries.items(): if dev == 'lo': diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py index 37cf93bf..c2d1f77d 100644 --- a/cloudinit/distros/bsd.py +++ b/cloudinit/distros/bsd.py @@ -5,6 +5,7 @@ from cloudinit.distros import bsd_utils from cloudinit import helpers from cloudinit import log as logging from cloudinit import net +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -50,7 +51,7 @@ class BSD(distros.Distro): else: group_add_cmd = self.group_add_cmd_prefix + [name] try: - util.subp(group_add_cmd) + subp.subp(group_add_cmd) LOG.info("Created new group %s", name) except Exception: util.logexc(LOG, "Failed to create group %s", name) @@ -63,7 +64,7 @@ class BSD(distros.Distro): "; user does not exist.", member, name) continue try: - util.subp(self._get_add_member_to_group_cmd(member, name)) + subp.subp(self._get_add_member_to_group_cmd(member, name)) LOG.info("Added user '%s' to group '%s'", member, name) except Exception: util.logexc(LOG, "Failed to add user '%s' to group '%s'", @@ -111,7 +112,7 @@ class BSD(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, env=self._get_pkg_cmd_environ(), capture=False) + subp.subp(cmd, env=self._get_pkg_cmd_environ(), capture=False) def _write_network_config(self, netconfig): return self._supported_write_network_config(netconfig) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 128bb523..844aaf21 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -13,6 +13,7 @@ import os from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros.parsers.hostname import HostnameConf @@ -197,7 +198,7 @@ class Distro(distros.Distro): # Allow the output of this to flow outwards (ie not be captured) util.log_time(logfunc=LOG.debug, msg="apt-%s [%s]" % (command, ' '.join(cmd)), - func=util.subp, + func=subp.subp, args=(cmd,), kwargs={'env': e, 'capture': False}) def update_package_sources(self): @@ -214,7 +215,7 @@ def _get_wrapper_prefix(cmd, mode): if (util.is_true(mode) or (str(mode).lower() == "auto" and cmd[0] and - util.which(cmd[0]))): + subp.which(cmd[0]))): return cmd else: return [] @@ -269,7 +270,7 @@ def update_locale_conf(locale, sys_path, keyname='LANG'): """Update system locale config""" LOG.debug('Updating %s with locale setting %s=%s', sys_path, keyname, locale) - util.subp( + subp.subp( ['update-locale', '--locale-file=' + sys_path, '%s=%s' % (keyname, locale)], capture=False) @@ -291,7 +292,7 @@ def regenerate_locale(locale, sys_path, keyname='LANG'): # finally, trigger regeneration LOG.debug('Generating locales for %s', locale) - util.subp(['locale-gen', locale], capture=False) + subp.subp(['locale-gen', locale], capture=False) # vi: ts=4 expandtab diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index b3a4ad67..dde34d41 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -10,6 +10,7 @@ from io import StringIO import cloudinit.distros.bsd from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -78,7 +79,7 @@ class Distro(cloudinit.distros.bsd.BSD): # Run the command LOG.info("Adding user %s", name) try: - util.subp(pw_useradd_cmd, logstring=log_pw_useradd_cmd) + subp.subp(pw_useradd_cmd, logstring=log_pw_useradd_cmd) except Exception: util.logexc(LOG, "Failed to create user %s", name) raise @@ -90,7 +91,7 @@ class Distro(cloudinit.distros.bsd.BSD): def expire_passwd(self, user): try: - util.subp(['pw', 'usermod', user, '-p', '01-Jan-1970']) + subp.subp(['pw', 'usermod', user, '-p', '01-Jan-1970']) except Exception: util.logexc(LOG, "Failed to set pw expiration for %s", user) raise @@ -102,7 +103,7 @@ class Distro(cloudinit.distros.bsd.BSD): hash_opt = "-h" try: - util.subp(['pw', 'usermod', user, hash_opt, '0'], + subp.subp(['pw', 'usermod', user, hash_opt, '0'], data=passwd, logstring="chpasswd for %s" % user) except Exception: util.logexc(LOG, "Failed to set password for %s", user) @@ -110,7 +111,7 @@ class Distro(cloudinit.distros.bsd.BSD): def lock_passwd(self, name): try: - util.subp(['pw', 'usermod', name, '-h', '-']) + subp.subp(['pw', 'usermod', name, '-h', '-']) except Exception: util.logexc(LOG, "Failed to lock user %s", name) raise @@ -131,8 +132,8 @@ class Distro(cloudinit.distros.bsd.BSD): try: LOG.debug("Running cap_mkdb for %s", locale) - util.subp(['cap_mkdb', self.login_conf_fn]) - except util.ProcessExecutionError: + subp.subp(['cap_mkdb', self.login_conf_fn]) + except subp.ProcessExecutionError: # cap_mkdb failed, so restore the backup. util.logexc(LOG, "Failed to apply locale %s", locale) try: diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index dc57717d..2bee1c89 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -9,6 +9,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros import net_util @@ -39,7 +40,7 @@ class Distro(distros.Distro): def apply_locale(self, locale, out_fn=None): if not out_fn: out_fn = self.locale_conf_fn - util.subp(['locale-gen', '-G', locale], capture=False) + subp.subp(['locale-gen', '-G', locale], capture=False) # "" provides trailing newline during join lines = [ util.make_header(), @@ -94,11 +95,11 @@ class Distro(distros.Distro): cmd = ['rc-update', 'add', 'net.{name}'.format(name=dev), 'default'] try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) @@ -119,12 +120,12 @@ class Distro(distros.Distro): LOG.debug("Attempting to run bring up interface %s using command %s", device_name, cmd) try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False @@ -137,11 +138,11 @@ class Distro(distros.Distro): # Grab device names from init scripts cmd = ['ls', '/etc/init.d/net.*'] try: - (_out, err) = util.subp(cmd) + (_out, err) = subp.subp(cmd) if len(err): LOG.warning("Running %s resulted in stderr output: %s", cmd, err) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False devices = [x.split('.')[2] for x in _out.split(' ')] @@ -208,7 +209,7 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def update_package_sources(self): self._runner.run("update-sources", self.package_command, diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py index ecc8239a..066737a8 100644 --- a/cloudinit/distros/netbsd.py +++ b/cloudinit/distros/netbsd.py @@ -8,6 +8,7 @@ import platform import cloudinit.distros.bsd from cloudinit import log as logging +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -76,7 +77,7 @@ class NetBSD(cloudinit.distros.bsd.BSD): # Run the command LOG.info("Adding user %s", name) try: - util.subp(adduser_cmd, logstring=log_adduser_cmd) + subp.subp(adduser_cmd, logstring=log_adduser_cmd) except Exception: util.logexc(LOG, "Failed to create user %s", name) raise @@ -103,7 +104,7 @@ class NetBSD(cloudinit.distros.bsd.BSD): crypt.mksalt(method)) try: - util.subp(['usermod', '-p', hashed_pw, user]) + subp.subp(['usermod', '-p', hashed_pw, user]) except Exception: util.logexc(LOG, "Failed to set password for %s", user) raise @@ -111,21 +112,21 @@ class NetBSD(cloudinit.distros.bsd.BSD): def force_passwd_change(self, user): try: - util.subp(['usermod', '-F', user]) + subp.subp(['usermod', '-F', user]) except Exception: util.logexc(LOG, "Failed to set pw expiration for %s", user) raise def lock_passwd(self, name): try: - util.subp(['usermod', '-C', 'yes', name]) + subp.subp(['usermod', '-C', 'yes', name]) except Exception: util.logexc(LOG, "Failed to lock user %s", name) raise def unlock_passwd(self, name): try: - util.subp(['usermod', '-C', 'no', name]) + subp.subp(['usermod', '-C', 'no', name]) except Exception: util.logexc(LOG, "Failed to unlock user %s", name) raise diff --git a/cloudinit/distros/openbsd.py b/cloudinit/distros/openbsd.py index ca094156..07c76530 100644 --- a/cloudinit/distros/openbsd.py +++ b/cloudinit/distros/openbsd.py @@ -7,6 +7,7 @@ import platform import cloudinit.distros.netbsd from cloudinit import log as logging +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -27,7 +28,7 @@ class Distro(cloudinit.distros.netbsd.NetBSD): def lock_passwd(self, name): try: - util.subp(['usermod', '-p', '*', name]) + subp.subp(['usermod', '-p', '*', name]) except Exception: util.logexc(LOG, "Failed to lock user %s", name) raise diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py index 68028d20..ffb7d0e8 100644 --- a/cloudinit/distros/opensuse.py +++ b/cloudinit/distros/opensuse.py @@ -14,6 +14,7 @@ from cloudinit.distros.parsers.hostname import HostnameConf from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros import rhel_util as rhutil @@ -97,7 +98,7 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def set_timezone(self, tz): tz_file = self._find_tz_file(tz) @@ -129,7 +130,7 @@ class Distro(distros.Distro): if self.uses_systemd() and filename.endswith('/previous-hostname'): return util.load_file(filename).strip() elif self.uses_systemd(): - (out, _err) = util.subp(['hostname']) + (out, _err) = subp.subp(['hostname']) if len(out): return out else: @@ -163,7 +164,7 @@ class Distro(distros.Distro): if self.uses_systemd() and out_fn.endswith('/previous-hostname'): util.write_file(out_fn, hostname) elif self.uses_systemd(): - util.subp(['hostnamectl', 'set-hostname', str(hostname)]) + subp.subp(['hostnamectl', 'set-hostname', str(hostname)]) else: conf = None try: diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index f55d96f7..c72f7c17 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -11,6 +11,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.distros import rhel_util @@ -83,7 +84,7 @@ class Distro(distros.Distro): if self.uses_systemd() and out_fn.endswith('/previous-hostname'): util.write_file(out_fn, hostname) elif self.uses_systemd(): - util.subp(['hostnamectl', 'set-hostname', str(hostname)]) + subp.subp(['hostnamectl', 'set-hostname', str(hostname)]) else: host_cfg = { 'HOSTNAME': hostname, @@ -108,7 +109,7 @@ class Distro(distros.Distro): if self.uses_systemd() and filename.endswith('/previous-hostname'): return util.load_file(filename).strip() elif self.uses_systemd(): - (out, _err) = util.subp(['hostname']) + (out, _err) = subp.subp(['hostname']) if len(out): return out else: @@ -146,7 +147,7 @@ class Distro(distros.Distro): if pkgs is None: pkgs = [] - if util.which('dnf'): + if subp.which('dnf'): LOG.debug('Using DNF for package management') cmd = ['dnf'] else: @@ -173,7 +174,7 @@ class Distro(distros.Distro): cmd.extend(pkglist) # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) + subp.subp(cmd, capture=False) def update_package_sources(self): self._runner.run("update-sources", self.package_command, diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py index 7fe17a2e..72b5ac59 100644 --- a/cloudinit/gpg.py +++ b/cloudinit/gpg.py @@ -8,7 +8,7 @@ """gpg.py - Collection of gpg key related functions""" from cloudinit import log as logging -from cloudinit import util +from cloudinit import subp import time @@ -18,9 +18,9 @@ LOG = logging.getLogger(__name__) def export_armour(key): """Export gpg key, armoured key gets returned""" try: - (armour, _) = util.subp(["gpg", "--export", "--armour", key], + (armour, _) = subp.subp(["gpg", "--export", "--armour", key], capture=True) - except util.ProcessExecutionError as error: + except subp.ProcessExecutionError as error: # debug, since it happens for any key not on the system initially LOG.debug('Failed to export armoured key "%s": %s', key, error) armour = None @@ -51,11 +51,11 @@ def recv_key(key, keyserver, retries=(1, 1)): while True: trynum += 1 try: - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) LOG.debug("Imported key '%s' from keyserver '%s' on try %d", key, keyserver, trynum) return - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: error = e try: naplen = next(sleeps) @@ -72,9 +72,9 @@ def recv_key(key, keyserver, retries=(1, 1)): def delete_key(key): """Delete the specified key from the local gpg ring""" try: - util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], + subp.subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=True) - except util.ProcessExecutionError as error: + except subp.ProcessExecutionError as error: LOG.warning('Failed delete key "%s": %s', key, error) diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index dca50a49..c6205097 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -12,6 +12,7 @@ import os from cloudinit import handlers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.settings import (PER_ALWAYS) @@ -48,8 +49,8 @@ class BootHookPartHandler(handlers.Handler): env = os.environ.copy() if self.instance_id is not None: env['INSTANCE_ID'] = str(self.instance_id) - util.subp([filepath], env=env) - except util.ProcessExecutionError: + subp.subp([filepath], env=env) + except subp.ProcessExecutionError: util.logexc(LOG, "Boothooks script %s execution error", filepath) except Exception: util.logexc(LOG, "Boothooks unknown error when running %s", diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index 003cad60..a9d29537 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -13,6 +13,7 @@ import re from cloudinit import handlers from cloudinit import log as logging +from cloudinit import subp from cloudinit import util from cloudinit.settings import (PER_INSTANCE) @@ -52,7 +53,7 @@ class UpstartJobPartHandler(handlers.Handler): util.write_file(path, payload, 0o644) if SUITABLE_UPSTART: - util.subp(["initctl", "reload-configuration"], capture=False) + subp.subp(["initctl", "reload-configuration"], capture=False) def _has_suitable_upstart(): @@ -63,7 +64,7 @@ def _has_suitable_upstart(): if not os.path.exists("/sbin/initctl"): return False try: - (version_out, _err) = util.subp(["initctl", "version"]) + (version_out, _err) = subp.subp(["initctl", "version"]) except Exception: util.logexc(LOG, "initctl version failed") return False @@ -77,7 +78,7 @@ def _has_suitable_upstart(): if not os.path.exists("/usr/bin/dpkg-query"): return False try: - (dpkg_ver, _err) = util.subp(["dpkg-query", + (dpkg_ver, _err) = subp.subp(["dpkg-query", "--showformat=${Version}", "--show", "upstart"], rcs=[0, 1]) except Exception: @@ -86,9 +87,9 @@ def _has_suitable_upstart(): try: good = "1.8-0ubuntu1.2" - util.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good]) + subp.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good]) return True - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code == 1: pass else: diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 8af24fa9..a57fea0a 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -12,6 +12,7 @@ import os import re from functools import partial +from cloudinit import subp from cloudinit import util from cloudinit.net.network_state import mask_to_net_prefix from cloudinit.url_helper import UrlError, readurl @@ -358,7 +359,7 @@ def find_fallback_nic_on_freebsd(blacklist_drivers=None): we'll use the first interface from ``ifconfig -l -u ether`` """ - stdout, _stderr = util.subp(['ifconfig', '-l', '-u', 'ether']) + stdout, _stderr = subp.subp(['ifconfig', '-l', '-u', 'ether']) values = stdout.split() if values: return values[0] @@ -620,9 +621,9 @@ def _get_current_rename_info(check_downable=True): if check_downable: nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]") - ipv6, _err = util.subp(['ip', '-6', 'addr', 'show', 'permanent', + ipv6, _err = subp.subp(['ip', '-6', 'addr', 'show', 'permanent', 'scope', 'global'], capture=True) - ipv4, _err = util.subp(['ip', '-4', 'addr', 'show'], capture=True) + ipv4, _err = subp.subp(['ip', '-4', 'addr', 'show'], capture=True) nics_with_addresses = set() for bytes_out in (ipv6, ipv4): @@ -658,13 +659,13 @@ def _rename_interfaces(renames, strict_present=True, strict_busy=True, for data in cur_info.values()) def rename(cur, new): - util.subp(["ip", "link", "set", cur, "name", new], capture=True) + subp.subp(["ip", "link", "set", cur, "name", new], capture=True) def down(name): - util.subp(["ip", "link", "set", name, "down"], capture=True) + subp.subp(["ip", "link", "set", name, "down"], capture=True) def up(name): - util.subp(["ip", "link", "set", name, "up"], capture=True) + subp.subp(["ip", "link", "set", name, "up"], capture=True) ops = [] errors = [] @@ -819,7 +820,7 @@ def get_interfaces_by_mac(): def get_interfaces_by_mac_on_freebsd(): - (out, _) = util.subp(['ifconfig', '-a', 'ether']) + (out, _) = subp.subp(['ifconfig', '-a', 'ether']) # flatten each interface block in a single line def flatten(out): @@ -850,7 +851,7 @@ def get_interfaces_by_mac_on_netbsd(): re_field_match = ( r"(?P\w+).*address:\s" r"(?P([\da-f]{2}[:-]){5}([\da-f]{2})).*") - (out, _) = util.subp(['ifconfig', '-a']) + (out, _) = subp.subp(['ifconfig', '-a']) if_lines = re.sub(r'\n\s+', ' ', out).splitlines() for line in if_lines: m = re.match(re_field_match, line) @@ -865,7 +866,7 @@ def get_interfaces_by_mac_on_openbsd(): re_field_match = ( r"(?P\w+).*lladdr\s" r"(?P([\da-f]{2}[:-]){5}([\da-f]{2})).*") - (out, _) = util.subp(['ifconfig', '-a']) + (out, _) = subp.subp(['ifconfig', '-a']) if_lines = re.sub(r'\n\s+', ' ', out).splitlines() for line in if_lines: m = re.match(re_field_match, line) @@ -1067,11 +1068,11 @@ class EphemeralIPv4Network(object): def __exit__(self, excp_type, excp_value, excp_traceback): """Teardown anything we set up.""" for cmd in self.cleanup_cmds: - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def _delete_address(self, address, prefix): """Perform the ip command to remove the specified address.""" - util.subp( + subp.subp( ['ip', '-family', 'inet', 'addr', 'del', '%s/%s' % (address, prefix), 'dev', self.interface], capture=True) @@ -1083,11 +1084,11 @@ class EphemeralIPv4Network(object): 'Attempting setup of ephemeral network on %s with %s brd %s', self.interface, cidr, self.broadcast) try: - util.subp( + subp.subp( ['ip', '-family', 'inet', 'addr', 'add', cidr, 'broadcast', self.broadcast, 'dev', self.interface], capture=True, update_env={'LANG': 'C'}) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if "File exists" not in e.stderr: raise LOG.debug( @@ -1095,7 +1096,7 @@ class EphemeralIPv4Network(object): self.interface, self.ip) else: # Address creation success, bring up device and queue cleanup - util.subp( + subp.subp( ['ip', '-family', 'inet', 'link', 'set', 'dev', self.interface, 'up'], capture=True) self.cleanup_cmds.append( @@ -1112,7 +1113,7 @@ class EphemeralIPv4Network(object): via_arg = [] if gateway != "0.0.0.0/0": via_arg = ['via', gateway] - util.subp( + subp.subp( ['ip', '-4', 'route', 'add', net_address] + via_arg + ['dev', self.interface], capture=True) self.cleanup_cmds.insert( @@ -1122,20 +1123,20 @@ class EphemeralIPv4Network(object): def _bringup_router(self): """Perform the ip commands to fully setup the router if needed.""" # Check if a default route exists and exit if it does - out, _ = util.subp(['ip', 'route', 'show', '0.0.0.0/0'], capture=True) + out, _ = subp.subp(['ip', 'route', 'show', '0.0.0.0/0'], capture=True) if 'default' in out: LOG.debug( 'Skip ephemeral route setup. %s already has default route: %s', self.interface, out.strip()) return - util.subp( + subp.subp( ['ip', '-4', 'route', 'add', self.router, 'dev', self.interface, 'src', self.ip], capture=True) self.cleanup_cmds.insert( 0, ['ip', '-4', 'route', 'del', self.router, 'dev', self.interface, 'src', self.ip]) - util.subp( + subp.subp( ['ip', '-4', 'route', 'add', 'default', 'via', self.router, 'dev', self.interface], capture=True) self.cleanup_cmds.insert( diff --git a/cloudinit/net/bsd.py b/cloudinit/net/bsd.py index fb714d4c..1c355a98 100644 --- a/cloudinit/net/bsd.py +++ b/cloudinit/net/bsd.py @@ -5,6 +5,7 @@ import re from cloudinit import log as logging from cloudinit import net from cloudinit import util +from cloudinit import subp from cloudinit.distros.parsers.resolv_conf import ResolvConf from cloudinit.distros import bsd_utils @@ -18,11 +19,11 @@ class BSDRenderer(renderer.Renderer): rc_conf_fn = 'etc/rc.conf' def get_rc_config_value(self, key): - fn = util.target_path(self.target, self.rc_conf_fn) + fn = subp.target_path(self.target, self.rc_conf_fn) bsd_utils.get_rc_config_value(key, fn=fn) def set_rc_config_value(self, key, value): - fn = util.target_path(self.target, self.rc_conf_fn) + fn = subp.target_path(self.target, self.rc_conf_fn) bsd_utils.set_rc_config_value(key, value, fn=fn) def __init__(self, config=None): @@ -111,12 +112,12 @@ class BSDRenderer(renderer.Renderer): # Try to read the /etc/resolv.conf or just start from scratch if that # fails. try: - resolvconf = ResolvConf(util.load_file(util.target_path( + resolvconf = ResolvConf(util.load_file(subp.target_path( target, self.resolv_conf_fn))) resolvconf.parse() except IOError: util.logexc(LOG, "Failed to parse %s, use new empty file", - util.target_path(target, self.resolv_conf_fn)) + subp.target_path(target, self.resolv_conf_fn)) resolvconf = ResolvConf('') resolvconf.parse() @@ -134,7 +135,7 @@ class BSDRenderer(renderer.Renderer): except ValueError: util.logexc(LOG, "Failed to add search domain %s", domain) util.write_file( - util.target_path(target, self.resolv_conf_fn), + subp.target_path(target, self.resolv_conf_fn), str(resolvconf), 0o644) def render_network_state(self, network_state, templates=None, target=None): diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 19d0199c..d03baeab 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -17,6 +17,7 @@ from cloudinit.net import ( has_url_connectivity) from cloudinit.net.network_state import mask_and_ipv4_to_bcast_addr as bcip from cloudinit import temp_utils +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -150,7 +151,7 @@ def maybe_perform_dhcp_discovery(nic=None): LOG.debug( 'Skip dhcp_discovery: nic %s not found in get_devicelist.', nic) return [] - dhclient_path = util.which('dhclient') + dhclient_path = subp.which('dhclient') if not dhclient_path: LOG.debug('Skip dhclient configuration: No dhclient command found.') return [] @@ -219,10 +220,10 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir): # Generally dhclient relies on dhclient-script PREINIT action to bring the # link up before attempting discovery. Since we are using -sf /bin/true, # we need to do that "link up" ourselves first. - util.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True) + subp.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True) cmd = [sandbox_dhclient_cmd, '-1', '-v', '-lf', lease_file, '-pf', pid_file, interface, '-sf', '/bin/true'] - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) # Wait for pid file and lease file to appear, and for the process # named by the pid file to daemonize (have pid 1 as its parent). If we diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index 2f714563..b4c69457 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -11,6 +11,7 @@ from . import renderer from .network_state import subnet_is_ipv6 from cloudinit import log as logging +from cloudinit import subp from cloudinit import util @@ -511,13 +512,13 @@ class Renderer(renderer.Renderer): return '\n\n'.join(['\n'.join(s) for s in sections]) + "\n" def render_network_state(self, network_state, templates=None, target=None): - fpeni = util.target_path(target, self.eni_path) + fpeni = subp.target_path(target, self.eni_path) util.ensure_dir(os.path.dirname(fpeni)) header = self.eni_header if self.eni_header else "" util.write_file(fpeni, header + self._render_interfaces(network_state)) if self.netrules_path: - netrules = util.target_path(target, self.netrules_path) + netrules = subp.target_path(target, self.netrules_path) util.ensure_dir(os.path.dirname(netrules)) util.write_file(netrules, self._render_persistent_net(network_state)) @@ -544,9 +545,9 @@ def available(target=None): expected = ['ifquery', 'ifup', 'ifdown'] search = ['/sbin', '/usr/sbin'] for p in expected: - if not util.which(p, search=search, target=target): + if not subp.which(p, search=search, target=target): return False - eni = util.target_path(target, 'etc/network/interfaces') + eni = subp.target_path(target, 'etc/network/interfaces') if not os.path.isfile(eni): return False diff --git a/cloudinit/net/freebsd.py b/cloudinit/net/freebsd.py index 60f05bb2..0285dfec 100644 --- a/cloudinit/net/freebsd.py +++ b/cloudinit/net/freebsd.py @@ -2,6 +2,7 @@ from cloudinit import log as logging import cloudinit.net.bsd +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -30,17 +31,17 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): LOG.debug("freebsd generate postcmd disabled") return - util.subp(['service', 'netif', 'restart'], capture=True) + subp.subp(['service', 'netif', 'restart'], capture=True) # On FreeBSD 10, the restart of routing and dhclient is likely to fail # because # - routing: it cannot remove the loopback route, but it will still set # up the default route as expected. # - dhclient: it cannot stop the dhclient started by the netif service. # In both case, the situation is ok, and we can proceed. - util.subp(['service', 'routing', 'restart'], capture=True, rcs=[0, 1]) + subp.subp(['service', 'routing', 'restart'], capture=True, rcs=[0, 1]) for dhcp_interface in self.dhcp_interfaces(): - util.subp(['service', 'dhclient', 'restart', dhcp_interface], + subp.subp(['service', 'dhclient', 'restart', dhcp_interface], rcs=[0, 1], capture=True) diff --git a/cloudinit/net/netbsd.py b/cloudinit/net/netbsd.py index 9cc8ef31..30437b5f 100644 --- a/cloudinit/net/netbsd.py +++ b/cloudinit/net/netbsd.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import log as logging +from cloudinit import subp from cloudinit import util import cloudinit.net.bsd @@ -29,9 +30,9 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): LOG.debug("netbsd generate postcmd disabled") return - util.subp(['service', 'network', 'restart'], capture=True) + subp.subp(['service', 'network', 'restart'], capture=True) if self.dhcp_interfaces(): - util.subp(['service', 'dhcpcd', 'restart'], capture=True) + subp.subp(['service', 'dhcpcd', 'restart'], capture=True) def set_route(self, network, netmask, gateway): if network == '0.0.0.0': diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 89855270..53347c83 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -8,6 +8,7 @@ from .network_state import subnet_is_ipv6, NET_CONFIG_TO_V2, IPV6_DYNAMIC_TYPES from cloudinit import log as logging from cloudinit import util +from cloudinit import subp from cloudinit import safeyaml from cloudinit.net import SYS_CLASS_NET, get_devicelist @@ -164,14 +165,14 @@ def _extract_bond_slaves_by_name(interfaces, entry, bond_master): def _clean_default(target=None): # clean out any known default files and derived files in target # LP: #1675576 - tpath = util.target_path(target, "etc/netplan/00-snapd-config.yaml") + tpath = subp.target_path(target, "etc/netplan/00-snapd-config.yaml") if not os.path.isfile(tpath): return content = util.load_file(tpath, decode=False) if content != KNOWN_SNAPD_CONFIG: return - derived = [util.target_path(target, f) for f in ( + derived = [subp.target_path(target, f) for f in ( 'run/systemd/network/10-netplan-all-en.network', 'run/systemd/network/10-netplan-all-eth.network', 'run/systemd/generator/netplan.stamp')] @@ -203,10 +204,10 @@ class Renderer(renderer.Renderer): def features(self): if self._features is None: try: - info_blob, _err = util.subp(self.NETPLAN_INFO, capture=True) + info_blob, _err = subp.subp(self.NETPLAN_INFO, capture=True) info = util.load_yaml(info_blob) self._features = info['netplan.io']['features'] - except util.ProcessExecutionError: + except subp.ProcessExecutionError: # if the info subcommand is not present then we don't have any # new features pass @@ -218,7 +219,7 @@ class Renderer(renderer.Renderer): # check network state for version # if v2, then extract network_state.config # else render_v2_from_state - fpnplan = os.path.join(util.target_path(target), self.netplan_path) + fpnplan = os.path.join(subp.target_path(target), self.netplan_path) util.ensure_dir(os.path.dirname(fpnplan)) header = self.netplan_header if self.netplan_header else "" @@ -239,7 +240,7 @@ class Renderer(renderer.Renderer): if not run: LOG.debug("netplan generate postcmd disabled") return - util.subp(self.NETPLAN_GENERATE, capture=True) + subp.subp(self.NETPLAN_GENERATE, capture=True) def _net_setup_link(self, run=False): """To ensure device link properties are applied, we poke @@ -253,7 +254,7 @@ class Renderer(renderer.Renderer): for cmd in [setup_lnk + [SYS_CLASS_NET + iface] for iface in get_devicelist() if os.path.islink(SYS_CLASS_NET + iface)]: - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) def _render_content(self, network_state): @@ -406,7 +407,7 @@ def available(target=None): expected = ['netplan'] search = ['/usr/sbin', '/sbin'] for p in expected: - if not util.which(p, search=search, target=target): + if not subp.which(p, search=search, target=target): return False return True diff --git a/cloudinit/net/openbsd.py b/cloudinit/net/openbsd.py index b9897e90..489ea48b 100644 --- a/cloudinit/net/openbsd.py +++ b/cloudinit/net/openbsd.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import log as logging +from cloudinit import subp from cloudinit import util import cloudinit.net.bsd @@ -12,7 +13,7 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): def write_config(self): for device_name, v in self.interface_configurations.items(): if_file = 'etc/hostname.{}'.format(device_name) - fn = util.target_path(self.target, if_file) + fn = subp.target_path(self.target, if_file) if device_name in self.dhcp_interfaces(): content = 'dhcp\n' elif isinstance(v, dict): @@ -30,12 +31,12 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): if not self._postcmds: LOG.debug("openbsd generate postcmd disabled") return - util.subp(['sh', '/etc/netstart'], capture=True) + subp.subp(['sh', '/etc/netstart'], capture=True) def set_route(self, network, netmask, gateway): if network == '0.0.0.0': if_file = 'etc/mygate' - fn = util.target_path(self.target, if_file) + fn = subp.target_path(self.target, if_file) content = gateway + '\n' util.write_file(fn, content) diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 0a387377..f36c300f 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -9,6 +9,7 @@ from configobj import ConfigObj from cloudinit import log as logging from cloudinit import util +from cloudinit import subp from cloudinit.distros.parsers import networkmanager_conf from cloudinit.distros.parsers import resolv_conf @@ -858,19 +859,19 @@ class Renderer(renderer.Renderer): if not templates: templates = self.templates file_mode = 0o644 - base_sysconf_dir = util.target_path(target, self.sysconf_dir) + base_sysconf_dir = subp.target_path(target, self.sysconf_dir) for path, data in self._render_sysconfig(base_sysconf_dir, network_state, self.flavor, templates=templates).items(): util.write_file(path, data, file_mode) if self.dns_path: - dns_path = util.target_path(target, self.dns_path) + dns_path = subp.target_path(target, self.dns_path) resolv_content = self._render_dns(network_state, existing_dns_path=dns_path) if resolv_content: util.write_file(dns_path, resolv_content, file_mode) if self.networkmanager_conf_path: - nm_conf_path = util.target_path(target, + nm_conf_path = subp.target_path(target, self.networkmanager_conf_path) nm_conf_content = self._render_networkmanager_conf(network_state, templates) @@ -878,12 +879,12 @@ class Renderer(renderer.Renderer): util.write_file(nm_conf_path, nm_conf_content, file_mode) if self.netrules_path: netrules_content = self._render_persistent_net(network_state) - netrules_path = util.target_path(target, self.netrules_path) + netrules_path = subp.target_path(target, self.netrules_path) util.write_file(netrules_path, netrules_content, file_mode) if available_nm(target=target): - enable_ifcfg_rh(util.target_path(target, path=NM_CFG_FILE)) + enable_ifcfg_rh(subp.target_path(target, path=NM_CFG_FILE)) - sysconfig_path = util.target_path(target, templates.get('control')) + sysconfig_path = subp.target_path(target, templates.get('control')) # Distros configuring /etc/sysconfig/network as a file e.g. Centos if sysconfig_path.endswith('network'): util.ensure_dir(os.path.dirname(sysconfig_path)) @@ -906,20 +907,20 @@ def available_sysconfig(target=None): expected = ['ifup', 'ifdown'] search = ['/sbin', '/usr/sbin'] for p in expected: - if not util.which(p, search=search, target=target): + if not subp.which(p, search=search, target=target): return False expected_paths = [ 'etc/sysconfig/network-scripts/network-functions', 'etc/sysconfig/config'] for p in expected_paths: - if os.path.isfile(util.target_path(target, p)): + if os.path.isfile(subp.target_path(target, p)): return True return False def available_nm(target=None): - if not os.path.isfile(util.target_path(target, path=NM_CFG_FILE)): + if not os.path.isfile(subp.target_path(target, path=NM_CFG_FILE)): return False return True diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index 7768da7c..d4881592 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -266,7 +266,7 @@ class TestDHCPDiscoveryClean(CiTestCase): 'Skip dhcp_discovery: nic idontexist not found in get_devicelist.', self.logs.getvalue()) - @mock.patch('cloudinit.net.dhcp.util.which') + @mock.patch('cloudinit.net.dhcp.subp.which') @mock.patch('cloudinit.net.dhcp.find_fallback_nic') def test_absent_dhclient_command(self, m_fallback, m_which): """When dhclient doesn't exist in the OS, log the issue and no-op.""" @@ -279,7 +279,7 @@ class TestDHCPDiscoveryClean(CiTestCase): @mock.patch('cloudinit.temp_utils.os.getuid') @mock.patch('cloudinit.net.dhcp.dhcp_discovery') - @mock.patch('cloudinit.net.dhcp.util.which') + @mock.patch('cloudinit.net.dhcp.subp.which') @mock.patch('cloudinit.net.dhcp.find_fallback_nic') def test_dhclient_run_with_tmpdir(self, m_fback, m_which, m_dhcp, m_uid): """maybe_perform_dhcp_discovery passes tmpdir to dhcp_discovery.""" @@ -302,7 +302,7 @@ class TestDHCPDiscoveryClean(CiTestCase): @mock.patch('time.sleep', mock.MagicMock()) @mock.patch('cloudinit.net.dhcp.os.kill') - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') def test_dhcp_discovery_run_in_sandbox_warns_invalid_pid(self, m_subp, m_kill): """dhcp_discovery logs a warning when pidfile contains invalid content. @@ -337,7 +337,7 @@ class TestDHCPDiscoveryClean(CiTestCase): @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid') @mock.patch('cloudinit.net.dhcp.os.kill') @mock.patch('cloudinit.net.dhcp.util.wait_for_files') - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') def test_dhcp_discovery_run_in_sandbox_waits_on_lease_and_pid(self, m_subp, m_wait, @@ -364,7 +364,7 @@ class TestDHCPDiscoveryClean(CiTestCase): @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid') @mock.patch('cloudinit.net.dhcp.os.kill') - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') def test_dhcp_discovery_run_in_sandbox(self, m_subp, m_kill, m_getppid): """dhcp_discovery brings up the interface and runs dhclient. @@ -529,7 +529,7 @@ class TestEphemeralDhcpNoNetworkSetup(HttprettyTestCase): # Ensure that no teardown happens: m_dhcp.assert_not_called() - @mock.patch('cloudinit.net.dhcp.util.subp') + @mock.patch('cloudinit.net.dhcp.subp.subp') @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') def test_ephemeral_dhcp_setup_network_if_url_connectivity( self, m_dhcp, m_subp): diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index 835ed807..1c0a16a8 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -14,7 +14,8 @@ import requests import cloudinit.net as net from cloudinit import safeyaml as yaml from cloudinit.tests.helpers import CiTestCase, HttprettyTestCase -from cloudinit.util import ProcessExecutionError, ensure_file, write_file +from cloudinit.subp import ProcessExecutionError +from cloudinit.util import ensure_file, write_file class TestSysDevPath(CiTestCase): @@ -541,7 +542,7 @@ class TestInterfaceHasOwnMAC(CiTestCase): net.interface_has_own_mac('eth1', strict=True) -@mock.patch('cloudinit.net.util.subp') +@mock.patch('cloudinit.net.subp.subp') class TestEphemeralIPV4Network(CiTestCase): with_logs = True diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 1001f149..628e2908 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -13,6 +13,7 @@ import re from cloudinit import log as logging from cloudinit.net.network_state import net_prefix_to_ipv4_mask +from cloudinit import subp from cloudinit import util from cloudinit.simpletable import SimpleTable @@ -197,15 +198,15 @@ def _netdev_info_ifconfig(ifconfig_data): def netdev_info(empty=""): devs = {} if util.is_NetBSD(): - (ifcfg_out, _err) = util.subp(["ifconfig", "-a"], rcs=[0, 1]) + (ifcfg_out, _err) = subp.subp(["ifconfig", "-a"], rcs=[0, 1]) devs = _netdev_info_ifconfig_netbsd(ifcfg_out) - elif util.which('ip'): + elif subp.which('ip'): # Try iproute first of all - (ipaddr_out, _err) = util.subp(["ip", "addr", "show"]) + (ipaddr_out, _err) = subp.subp(["ip", "addr", "show"]) devs = _netdev_info_iproute(ipaddr_out) - elif util.which('ifconfig'): + elif subp.which('ifconfig'): # Fall back to net-tools if iproute2 is not present - (ifcfg_out, _err) = util.subp(["ifconfig", "-a"], rcs=[0, 1]) + (ifcfg_out, _err) = subp.subp(["ifconfig", "-a"], rcs=[0, 1]) devs = _netdev_info_ifconfig(ifcfg_out) else: LOG.warning( @@ -285,10 +286,10 @@ def _netdev_route_info_iproute(iproute_data): entry['flags'] = ''.join(flags) routes['ipv4'].append(entry) try: - (iproute_data6, _err6) = util.subp( + (iproute_data6, _err6) = subp.subp( ["ip", "--oneline", "-6", "route", "list", "table", "all"], rcs=[0, 1]) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass else: entries6 = iproute_data6.splitlines() @@ -357,9 +358,9 @@ def _netdev_route_info_netstat(route_data): routes['ipv4'].append(entry) try: - (route_data6, _err6) = util.subp( + (route_data6, _err6) = subp.subp( ["netstat", "-A", "inet6", "--route", "--numeric"], rcs=[0, 1]) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass else: entries6 = route_data6.splitlines() @@ -393,13 +394,13 @@ def _netdev_route_info_netstat(route_data): def route_info(): routes = {} - if util.which('ip'): + if subp.which('ip'): # Try iproute first of all - (iproute_out, _err) = util.subp(["ip", "-o", "route", "list"]) + (iproute_out, _err) = subp.subp(["ip", "-o", "route", "list"]) routes = _netdev_route_info_iproute(iproute_out) - elif util.which('netstat'): + elif subp.which('netstat'): # Fall back to net-tools if iproute2 is not present - (route_out, _err) = util.subp( + (route_out, _err) = subp.subp( ["netstat", "--route", "--numeric", "--extend"], rcs=[0, 1]) routes = _netdev_route_info_netstat(route_out) else: diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index 5270fda8..ac3ecc3d 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -18,9 +18,9 @@ import os.path from cloudinit import log as logging from cloudinit import sources +from cloudinit import subp from cloudinit import util -from cloudinit.util import ProcessExecutionError LOG = logging.getLogger(__name__) @@ -192,7 +192,7 @@ class DataSourceAltCloud(sources.DataSource): # modprobe floppy try: modprobe_floppy() - except ProcessExecutionError as e: + except subp.ProcessExecutionError as e: util.logexc(LOG, 'Failed modprobe: %s', e) return False @@ -201,7 +201,7 @@ class DataSourceAltCloud(sources.DataSource): # udevadm settle for floppy device try: util.udevadm_settle(exists=floppy_dev, timeout=5) - except (ProcessExecutionError, OSError) as e: + except (subp.ProcessExecutionError, OSError) as e: util.logexc(LOG, 'Failed udevadm_settle: %s\n', e) return False @@ -261,7 +261,7 @@ class DataSourceAltCloud(sources.DataSource): def modprobe_floppy(): - out, _err = util.subp(CMD_PROBE_FLOPPY) + out, _err = subp.subp(CMD_PROBE_FLOPPY) LOG.debug('Command: %s\nOutput%s', ' '.join(CMD_PROBE_FLOPPY), out) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 01d9adf2..89312b9e 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -22,6 +22,7 @@ from cloudinit.event import EventType from cloudinit.net.dhcp import EphemeralDHCPv4 from cloudinit import sources from cloudinit.sources.helpers import netlink +from cloudinit import subp from cloudinit.url_helper import UrlError, readurl, retry_on_url_exc from cloudinit import util from cloudinit.reporting import events @@ -139,8 +140,8 @@ def find_dev_from_busdev(camcontrol_out, busdev): def execute_or_debug(cmd, fail_ret=None): try: - return util.subp(cmd)[0] - except util.ProcessExecutionError: + return subp.subp(cmd)[0] + except subp.ProcessExecutionError: LOG.debug("Failed to execute: %s", ' '.join(cmd)) return fail_ret @@ -252,11 +253,11 @@ DEF_PASSWD_REDACTION = 'REDACTED' def get_hostname(hostname_command='hostname'): if not isinstance(hostname_command, (list, tuple)): hostname_command = (hostname_command,) - return util.subp(hostname_command, capture=True)[0].strip() + return subp.subp(hostname_command, capture=True)[0].strip() def set_hostname(hostname, hostname_command='hostname'): - util.subp([hostname_command, hostname]) + subp.subp([hostname_command, hostname]) @azure_ds_telemetry_reporter @@ -343,7 +344,7 @@ class DataSourceAzure(sources.DataSource): try: invoke_agent(agent_cmd) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: # claim the datasource even if the command failed util.logexc(LOG, "agent command '%s' failed.", self.ds_cfg['agent_command']) @@ -982,7 +983,7 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname): if command == "builtin": if util.is_FreeBSD(): command = BOUNCE_COMMAND_FREEBSD - elif util.which('ifup'): + elif subp.which('ifup'): command = BOUNCE_COMMAND_IFUP else: LOG.debug( @@ -993,7 +994,7 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname): shell = not isinstance(command, (list, tuple)) # capture=False, see comments in bug 1202758 and bug 1206164. util.log_time(logfunc=LOG.debug, msg="publishing hostname", - get_uptime=True, func=util.subp, + get_uptime=True, func=subp.subp, kwargs={'args': command, 'shell': shell, 'capture': False, 'env': env}) return True @@ -1003,7 +1004,7 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname): def crtfile_to_pubkey(fname, data=None): pipeline = ('openssl x509 -noout -pubkey < "$0" |' 'ssh-keygen -i -m PKCS8 -f /dev/stdin') - (out, _err) = util.subp(['sh', '-c', pipeline, fname], + (out, _err) = subp.subp(['sh', '-c', pipeline, fname], capture=True, data=data) return out.rstrip() @@ -1015,7 +1016,7 @@ def pubkeys_from_crt_files(flist): for fname in flist: try: pubkeys.append(crtfile_to_pubkey(fname)) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: errors.append(fname) if errors: @@ -1057,7 +1058,7 @@ def invoke_agent(cmd): # this is a function itself to simplify patching it for test if cmd: LOG.debug("invoking agent: %s", cmd) - util.subp(cmd, shell=(not isinstance(cmd, list))) + subp.subp(cmd, shell=(not isinstance(cmd, list))) else: LOG.debug("not invoking agent") diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 2013bed7..54810439 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -22,6 +22,7 @@ from cloudinit import log as logging from cloudinit.net import dhcp from cloudinit import sources from cloudinit import url_helper as uhelp +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -46,7 +47,7 @@ class CloudStackPasswordServerClient(object): # The password server was in the past, a broken HTTP server, but is now # fixed. wget handles this seamlessly, so it's easier to shell out to # that rather than write our own handling code. - output, _ = util.subp([ + output, _ = subp.subp([ 'wget', '--quiet', '--tries', '3', '--timeout', '20', '--output-document', '-', '--header', 'DomU_Request: {0}'.format(domu_request), diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index ae31934b..62756cf7 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -10,6 +10,7 @@ import os from cloudinit import log as logging from cloudinit import sources +from cloudinit import subp from cloudinit import util from cloudinit.net import eni @@ -245,7 +246,7 @@ def find_candidate_devs(probe_optical=True, dslist=None): for device in OPTICAL_DEVICES: try: util.find_devs_with(path=device) - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass by_fstype = [] diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py index e0c714e8..d2aa9a58 100644 --- a/cloudinit/sources/DataSourceIBMCloud.py +++ b/cloudinit/sources/DataSourceIBMCloud.py @@ -99,6 +99,7 @@ import os from cloudinit import log as logging from cloudinit import sources from cloudinit.sources.helpers import openstack +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -240,7 +241,7 @@ def get_ibm_platform(): fslabels = {} try: devs = util.blkid() - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: LOG.warning("Failed to run blkid: %s", e) return (None, None) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 41f999e3..4c5eee4f 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -16,6 +16,7 @@ from xml.dom import minidom from cloudinit import log as logging from cloudinit import sources +from cloudinit import subp from cloudinit import util from cloudinit.sources.helpers.vmware.imc.config \ import Config @@ -536,15 +537,15 @@ def transport_iso9660(require_iso=True): def transport_vmware_guestinfo(): rpctool = "vmware-rpctool" not_found = None - if not util.which(rpctool): + if not subp.which(rpctool): return not_found cmd = [rpctool, "info-get guestinfo.ovfEnv"] try: - out, _err = util.subp(cmd) + out, _err = subp.subp(cmd) if out: return out LOG.debug("cmd %s exited 0 with empty stdout: %s", cmd, out) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: if e.exit_code != 1: LOG.warning("%s exited with code %d", rpctool, e.exit_code) LOG.debug(e) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index a08ab404..c7fdc2a5 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -21,6 +21,7 @@ import string from cloudinit import log as logging from cloudinit import net from cloudinit import sources +from cloudinit import subp from cloudinit import util @@ -334,7 +335,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None, cmd.extend(bash) - (output, _error) = util.subp(cmd, data=bcmd) + (output, _error) = subp.subp(cmd, data=bcmd) # exclude vars in bash that change on their own or that we used excluded = ( @@ -396,7 +397,7 @@ def read_context_disk_dir(source_dir, asuser=None): path = os.path.join(source_dir, 'context.sh') content = util.load_file(path) context = parse_shell_config(content, asuser=asuser) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: raise BrokenContextDiskDir("Error processing context.sh: %s" % (e)) except IOError as e: raise NonContextDiskDir("Error reading context.sh: %s" % (e)) diff --git a/cloudinit/sources/DataSourceRbxCloud.py b/cloudinit/sources/DataSourceRbxCloud.py index 084cb7d5..38fb5421 100644 --- a/cloudinit/sources/DataSourceRbxCloud.py +++ b/cloudinit/sources/DataSourceRbxCloud.py @@ -15,6 +15,7 @@ import os.path from cloudinit import log as logging from cloudinit import sources +from cloudinit import subp from cloudinit import util from cloudinit.event import EventType @@ -43,11 +44,11 @@ def int2ip(addr): def _sub_arp(cmd): """ - Uses the prefered cloud-init subprocess def of util.subp + Uses the prefered cloud-init subprocess def of subp.subp and runs arping. Breaking this to a separate function for later use in mocking and unittests """ - return util.subp(['arping'] + cmd) + return subp.subp(['arping'] + cmd) def gratuitous_arp(items, distro): @@ -61,7 +62,7 @@ def gratuitous_arp(items, distro): source_param, item['source'], item['destination'] ]) - except util.ProcessExecutionError as error: + except subp.ProcessExecutionError as error: # warning, because the system is able to function properly # despite no success - some ARP table may be waiting for # expiration, but the system may continue diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index cf676504..843b3a2a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -33,6 +33,7 @@ import socket from cloudinit import log as logging from cloudinit import serial from cloudinit import sources +from cloudinit import subp from cloudinit import util from cloudinit.event import EventType @@ -696,9 +697,9 @@ def identify_file(content_f): cmd = ["file", "--brief", "--mime-type", content_f] f_type = None try: - (f_type, _err) = util.subp(cmd) + (f_type, _err) = subp.subp(cmd) LOG.debug("script %s mime type is %s", content_f, f_type) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: util.logexc( LOG, ("Failed to identify script type for %s" % content_f, e)) return None if f_type is None else f_type.strip() diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 82b6730c..7bace8ca 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -15,6 +15,7 @@ from cloudinit import temp_utils from contextlib import contextmanager from xml.etree import ElementTree +from cloudinit import subp from cloudinit import url_helper from cloudinit import util from cloudinit import version @@ -92,7 +93,7 @@ def get_boot_telemetry(): raise RuntimeError("Failed to determine kernel start timestamp") try: - out, _ = util.subp(['/bin/systemctl', + out, _ = subp.subp(['/bin/systemctl', 'show', '-p', 'UserspaceTimestampMonotonic'], capture=True) @@ -105,7 +106,7 @@ def get_boot_telemetry(): "UserspaceTimestampMonotonic from systemd") user_start = kernel_start + (float(tsm) / 1000000) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: raise RuntimeError("Failed to get UserspaceTimestampMonotonic: %s" % e) except ValueError as e: @@ -114,7 +115,7 @@ def get_boot_telemetry(): % e) try: - out, _ = util.subp(['/bin/systemctl', 'show', + out, _ = subp.subp(['/bin/systemctl', 'show', 'cloud-init-local', '-p', 'InactiveExitTimestampMonotonic'], capture=True) @@ -126,7 +127,7 @@ def get_boot_telemetry(): "InactiveExitTimestampMonotonic from systemd") cloudinit_activation = kernel_start + (float(tsm) / 1000000) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: raise RuntimeError("Failed to get InactiveExitTimestampMonotonic: %s" % e) except ValueError as e: @@ -284,7 +285,7 @@ class OpenSSLManager(object): LOG.debug('Certificate already generated.') return with cd(self.tmpdir): - util.subp([ + subp.subp([ 'openssl', 'req', '-x509', '-nodes', '-subj', '/CN=LinuxTransport', '-days', '32768', '-newkey', 'rsa:2048', '-keyout', self.certificate_names['private_key'], @@ -301,14 +302,14 @@ class OpenSSLManager(object): @azure_ds_telemetry_reporter def _run_x509_action(action, cert): cmd = ['openssl', 'x509', '-noout', action] - result, _ = util.subp(cmd, data=cert) + result, _ = subp.subp(cmd, data=cert) return result @azure_ds_telemetry_reporter def _get_ssh_key_from_cert(self, certificate): pub_key = self._run_x509_action('-pubkey', certificate) keygen_cmd = ['ssh-keygen', '-i', '-m', 'PKCS8', '-f', '/dev/stdin'] - ssh_key, _ = util.subp(keygen_cmd, data=pub_key) + ssh_key, _ = subp.subp(keygen_cmd, data=pub_key) return ssh_key @azure_ds_telemetry_reporter @@ -341,7 +342,7 @@ class OpenSSLManager(object): certificates_content.encode('utf-8'), ] with cd(self.tmpdir): - out, _ = util.subp( + out, _ = subp.subp( 'openssl cms -decrypt -in /dev/stdin -inkey' ' {private_key} -recip {certificate} | openssl pkcs12 -nodes' ' -password pass:'.format(**self.certificate_names), diff --git a/cloudinit/sources/helpers/digitalocean.py b/cloudinit/sources/helpers/digitalocean.py index 0e7cccac..f5bbe46a 100644 --- a/cloudinit/sources/helpers/digitalocean.py +++ b/cloudinit/sources/helpers/digitalocean.py @@ -8,6 +8,7 @@ import random from cloudinit import log as logging from cloudinit import net as cloudnet from cloudinit import url_helper +from cloudinit import subp from cloudinit import util NIC_MAP = {'public': 'eth0', 'private': 'eth1'} @@ -36,14 +37,14 @@ def assign_ipv4_link_local(nic=None): ip_addr_cmd = ['ip', 'addr', 'add', addr, 'dev', nic] ip_link_cmd = ['ip', 'link', 'set', 'dev', nic, 'up'] - if not util.which('ip'): + if not subp.which('ip'): raise RuntimeError("No 'ip' command available to configure ip4LL " "address") try: - util.subp(ip_addr_cmd) + subp.subp(ip_addr_cmd) LOG.debug("assigned ip4LL address '%s' to '%s'", addr, nic) - util.subp(ip_link_cmd) + subp.subp(ip_link_cmd) LOG.debug("brought device '%s' up", nic) except Exception: util.logexc(LOG, "ip4LL address assignment of '%s' to '%s' failed." @@ -74,7 +75,7 @@ def del_ipv4_link_local(nic=None): ip_addr_cmd = ['ip', 'addr', 'flush', 'dev', nic] try: - util.subp(ip_addr_cmd) + subp.subp(ip_addr_cmd) LOG.debug("removed ip4LL addresses from %s", nic) except Exception as e: diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index a4373f24..c538720a 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -16,6 +16,7 @@ from cloudinit import ec2_utils from cloudinit import log as logging from cloudinit import net from cloudinit import sources +from cloudinit import subp from cloudinit import url_helper from cloudinit import util from cloudinit.sources import BrokenMetadata @@ -110,7 +111,7 @@ class SourceMixin(object): dev_entries = util.find_devs_with(criteria) if dev_entries: device = dev_entries[0] - except util.ProcessExecutionError: + except subp.ProcessExecutionError: pass return device diff --git a/cloudinit/sources/helpers/vmware/imc/config_custom_script.py b/cloudinit/sources/helpers/vmware/imc/config_custom_script.py index 9f14770e..2ab22de9 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_custom_script.py +++ b/cloudinit/sources/helpers/vmware/imc/config_custom_script.py @@ -9,6 +9,7 @@ import logging import os import stat +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -61,7 +62,7 @@ class PreCustomScript(RunCustomScript): """Executing custom script with precustomization argument.""" LOG.debug("Executing pre-customization script") self.prepare_script() - util.subp([CustomScriptConstant.CUSTOM_SCRIPT, "precustomization"]) + subp.subp([CustomScriptConstant.CUSTOM_SCRIPT, "precustomization"]) class PostCustomScript(RunCustomScript): diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 77cbf3b6..3745a262 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -10,6 +10,7 @@ import os import re from cloudinit.net.network_state import mask_to_net_prefix +from cloudinit import subp from cloudinit import util logger = logging.getLogger(__name__) @@ -73,7 +74,7 @@ class NicConfigurator(object): The mac address(es) are in the lower case """ cmd = ['ip', 'addr', 'show'] - output, _err = util.subp(cmd) + output, _err = subp.subp(cmd) sections = re.split(r'\n\d+: ', '\n' + output)[1:] macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))' @@ -248,8 +249,8 @@ class NicConfigurator(object): logger.info('Clearing DHCP leases') # Ignore the return code 1. - util.subp(["pkill", "dhclient"], rcs=[0, 1]) - util.subp(["rm", "-f", "/var/lib/dhcp/*"]) + subp.subp(["pkill", "dhclient"], rcs=[0, 1]) + subp.subp(["rm", "-f", "/var/lib/dhcp/*"]) def configure(self, osfamily=None): """ diff --git a/cloudinit/sources/helpers/vmware/imc/config_passwd.py b/cloudinit/sources/helpers/vmware/imc/config_passwd.py index 8c91fa41..d16a7690 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_passwd.py +++ b/cloudinit/sources/helpers/vmware/imc/config_passwd.py @@ -9,6 +9,7 @@ import logging import os +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -56,10 +57,10 @@ class PasswordConfigurator(object): LOG.info('Expiring password.') for user in uidUserList: try: - util.subp(['passwd', '--expire', user]) - except util.ProcessExecutionError as e: + subp.subp(['passwd', '--expire', user]) + except subp.ProcessExecutionError as e: if os.path.exists('/usr/bin/chage'): - util.subp(['chage', '-d', '0', user]) + subp.subp(['chage', '-d', '0', user]) else: LOG.warning('Failed to expire password for %s with error: ' '%s', user, e) diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index c60a38d7..893b1365 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -10,7 +10,7 @@ import os import re import time -from cloudinit import util +from cloudinit import subp from .guestcust_event import GuestCustEventEnum from .guestcust_state import GuestCustStateEnum @@ -34,7 +34,7 @@ def send_rpc(rpc): try: logger.debug("Sending RPC command: %s", rpc) - (out, err) = util.subp(["vmware-rpctool", rpc], rcs=[0]) + (out, err) = subp.subp(["vmware-rpctool", rpc], rcs=[0]) # Remove the trailing newline in the output. if out: out = out.rstrip() @@ -128,7 +128,7 @@ def get_tools_config(section, key, defaultVal): not installed. """ - if not util.which('vmware-toolbox-cmd'): + if not subp.which('vmware-toolbox-cmd'): logger.debug( 'vmware-toolbox-cmd not installed, returning default value') return defaultVal @@ -137,7 +137,7 @@ def get_tools_config(section, key, defaultVal): cmd = ['vmware-toolbox-cmd', 'config', 'get', section, key] try: - (outText, _) = util.subp(cmd) + (outText, _) = subp.subp(cmd) m = re.match(r'([^=]+)=(.*)', outText) if m: retValue = m.group(2).strip() @@ -147,7 +147,7 @@ def get_tools_config(section, key, defaultVal): logger.debug( "Tools config: [%s] %s is not found, return default value: %s", section, key, retValue) - except util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: logger.error("Failed running %s[%s]", cmd, e.exit_code) logger.exception(e) diff --git a/cloudinit/subp.py b/cloudinit/subp.py index 0ad09306..f8400b1f 100644 --- a/cloudinit/subp.py +++ b/cloudinit/subp.py @@ -1,9 +1,11 @@ # This file is part of cloud-init. See LICENSE file for license information. """Common utility functions for interacting with subprocess.""" -# TODO move subp shellify and runparts related functions out of util.py - import logging +import os +import subprocess + +from errno import ENOEXEC LOG = logging.getLogger(__name__) @@ -54,4 +56,299 @@ def prepend_base_command(base_command, commands): return fixed_commands +class ProcessExecutionError(IOError): + + MESSAGE_TMPL = ('%(description)s\n' + 'Command: %(cmd)s\n' + 'Exit code: %(exit_code)s\n' + 'Reason: %(reason)s\n' + 'Stdout: %(stdout)s\n' + 'Stderr: %(stderr)s') + empty_attr = '-' + + def __init__(self, stdout=None, stderr=None, + exit_code=None, cmd=None, + description=None, reason=None, + errno=None): + if not cmd: + self.cmd = self.empty_attr + else: + self.cmd = cmd + + if not description: + if not exit_code and errno == ENOEXEC: + self.description = 'Exec format error. Missing #! in script?' + else: + self.description = 'Unexpected error while running command.' + else: + self.description = description + + if not isinstance(exit_code, int): + self.exit_code = self.empty_attr + else: + self.exit_code = exit_code + + if not stderr: + if stderr is None: + self.stderr = self.empty_attr + else: + self.stderr = stderr + else: + self.stderr = self._indent_text(stderr) + + if not stdout: + if stdout is None: + self.stdout = self.empty_attr + else: + self.stdout = stdout + else: + self.stdout = self._indent_text(stdout) + + if reason: + self.reason = reason + else: + self.reason = self.empty_attr + + self.errno = errno + message = self.MESSAGE_TMPL % { + 'description': self._ensure_string(self.description), + 'cmd': self._ensure_string(self.cmd), + 'exit_code': self._ensure_string(self.exit_code), + 'stdout': self._ensure_string(self.stdout), + 'stderr': self._ensure_string(self.stderr), + 'reason': self._ensure_string(self.reason), + } + IOError.__init__(self, message) + + def _ensure_string(self, text): + """ + if data is bytes object, decode + """ + return text.decode() if isinstance(text, bytes) else text + + def _indent_text(self, text, indent_level=8): + """ + indent text on all but the first line, allowing for easy to read output + """ + cr = '\n' + indent = ' ' * indent_level + # if input is bytes, return bytes + if isinstance(text, bytes): + cr = cr.encode() + indent = indent.encode() + # remove any newlines at end of text first to prevent unneeded blank + # line in output + return text.rstrip(cr).replace(cr, cr + indent) + + +def subp(args, data=None, rcs=None, env=None, capture=True, + combine_capture=False, shell=False, + logstring=False, decode="replace", target=None, update_env=None, + status_cb=None): + """Run a subprocess. + + :param args: command to run in a list. [cmd, arg1, arg2...] + :param data: input to the command, made available on its stdin. + :param rcs: + a list of allowed return codes. If subprocess exits with a value not + in this list, a ProcessExecutionError will be raised. By default, + data is returned as a string. See 'decode' parameter. + :param env: a dictionary for the command's environment. + :param capture: + boolean indicating if output should be captured. If True, then stderr + and stdout will be returned. If False, they will not be redirected. + :param combine_capture: + boolean indicating if stderr should be redirected to stdout. When True, + interleaved stderr and stdout will be returned as the first element of + a tuple, the second will be empty string or bytes (per decode). + if combine_capture is True, then output is captured independent of + the value of capture. + :param shell: boolean indicating if this should be run with a shell. + :param logstring: + the command will be logged to DEBUG. If it contains info that should + not be logged, then logstring will be logged instead. + :param decode: + if False, no decoding will be done and returned stdout and stderr will + be bytes. Other allowed values are 'strict', 'ignore', and 'replace'. + These values are passed through to bytes().decode() as the 'errors' + parameter. There is no support for decoding to other than utf-8. + :param target: + not supported, kwarg present only to make function signature similar + to curtin's subp. + :param update_env: + update the enviornment for this command with this dictionary. + this will not affect the current processes os.environ. + :param status_cb: + call this fuction with a single string argument before starting + and after finishing. + + :return + if not capturing, return is (None, None) + if capturing, stdout and stderr are returned. + if decode: + entries in tuple will be python2 unicode or python3 string + if not decode: + entries in tuple will be python2 string or python3 bytes + """ + + # not supported in cloud-init (yet), for now kept in the call signature + # to ease maintaining code shared between cloud-init and curtin + if target is not None: + raise ValueError("target arg not supported by cloud-init") + + if rcs is None: + rcs = [0] + + devnull_fp = None + + if update_env: + if env is None: + env = os.environ + env = env.copy() + env.update(update_env) + + if target_path(target) != "/": + args = ['chroot', target] + list(args) + + if status_cb: + command = ' '.join(args) if isinstance(args, list) else args + status_cb('Begin run command: {command}\n'.format(command=command)) + if not logstring: + LOG.debug(("Running command %s with allowed return codes %s" + " (shell=%s, capture=%s)"), + args, rcs, shell, 'combine' if combine_capture else capture) + else: + LOG.debug(("Running hidden command to protect sensitive " + "input/output logstring: %s"), logstring) + + stdin = None + stdout = None + stderr = None + if capture: + stdout = subprocess.PIPE + stderr = subprocess.PIPE + if combine_capture: + stdout = subprocess.PIPE + stderr = subprocess.STDOUT + if data is None: + # using devnull assures any reads get null, rather + # than possibly waiting on input. + devnull_fp = open(os.devnull) + stdin = devnull_fp + else: + stdin = subprocess.PIPE + if not isinstance(data, bytes): + data = data.encode() + + # Popen converts entries in the arguments array from non-bytes to bytes. + # When locale is unset it may use ascii for that encoding which can + # cause UnicodeDecodeErrors. (LP: #1751051) + if isinstance(args, bytes): + bytes_args = args + elif isinstance(args, str): + bytes_args = args.encode("utf-8") + else: + bytes_args = [ + x if isinstance(x, bytes) else x.encode("utf-8") + for x in args] + try: + sp = subprocess.Popen(bytes_args, stdout=stdout, + stderr=stderr, stdin=stdin, + env=env, shell=shell) + (out, err) = sp.communicate(data) + except OSError as e: + if status_cb: + status_cb('ERROR: End run command: invalid command provided\n') + raise ProcessExecutionError( + cmd=args, reason=e, errno=e.errno, + stdout="-" if decode else b"-", + stderr="-" if decode else b"-") + finally: + if devnull_fp: + devnull_fp.close() + + # Just ensure blank instead of none. + if capture or combine_capture: + if not out: + out = b'' + if not err: + err = b'' + if decode: + def ldecode(data, m='utf-8'): + if not isinstance(data, bytes): + return data + return data.decode(m, decode) + + out = ldecode(out) + err = ldecode(err) + + rc = sp.returncode + if rc not in rcs: + if status_cb: + status_cb( + 'ERROR: End run command: exit({code})\n'.format(code=rc)) + raise ProcessExecutionError(stdout=out, stderr=err, + exit_code=rc, + cmd=args) + if status_cb: + status_cb('End run command: exit({code})\n'.format(code=rc)) + return (out, err) + + +def target_path(target, path=None): + # return 'path' inside target, accepting target as None + if target in (None, ""): + target = "/" + elif not isinstance(target, str): + raise ValueError("Unexpected input for target: %s" % target) + else: + target = os.path.abspath(target) + # abspath("//") returns "//" specifically for 2 slashes. + if target.startswith("//"): + target = target[1:] + + if not path: + return target + + # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /. + while len(path) and path[0] == "/": + path = path[1:] + + return os.path.join(target, path) + + +def which(program, search=None, target=None): + target = target_path(target) + + if os.path.sep in program: + # if program had a '/' in it, then do not search PATH + # 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls + # so effectively we set cwd to / (or target) + if is_exe(target_path(target, program)): + return program + + if search is None: + paths = [p.strip('"') for p in + os.environ.get("PATH", "").split(os.pathsep)] + if target == "/": + search = paths + else: + search = [p for p in paths if p.startswith("/")] + + # normalize path input + search = [os.path.abspath(p) for p in search] + + for path in search: + ppath = os.path.sep.join((path, program)) + if is_exe(target_path(target, ppath)): + return ppath + + return None + + +def is_exe(fpath): + # return boolean indicating if fpath exists and is executable. + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + # vi: ts=4 expandtab diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index f3ab7e8c..58f63b69 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -23,9 +23,10 @@ from cloudinit import distros from cloudinit import helpers as ch from cloudinit.sources import DataSourceNone from cloudinit.templater import JINJA_AVAILABLE +from cloudinit import subp from cloudinit import util -_real_subp = util.subp +_real_subp = subp.subp # Used for skipping tests SkipTest = unittest.SkipTest @@ -134,9 +135,9 @@ class CiTestCase(TestCase): self.old_handlers = self.logger.handlers self.logger.handlers = [handler] if self.allowed_subp is True: - util.subp = _real_subp + subp.subp = _real_subp else: - util.subp = self._fake_subp + subp.subp = self._fake_subp def _fake_subp(self, *args, **kwargs): if 'args' in kwargs: @@ -171,7 +172,7 @@ class CiTestCase(TestCase): # Remove the handler we setup logging.getLogger().handlers = self.old_handlers logging.getLogger().level = None - util.subp = _real_subp + subp.subp = _real_subp super(CiTestCase, self).tearDown() def tmp_dir(self, dir=None, cleanup=True): @@ -280,13 +281,13 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): mock.patch.object(mod, f, trap_func)) # Handle subprocess calls - func = getattr(util, 'subp') + func = getattr(subp, 'subp') def nsubp(*_args, **_kwargs): return ('', '') self.patched_funcs.enter_context( - mock.patch.object(util, 'subp', nsubp)) + mock.patch.object(subp, 'subp', nsubp)) def null_func(*_args, **_kwargs): return None diff --git a/cloudinit/tests/test_conftest.py b/cloudinit/tests/test_conftest.py index 773ef8fe..a6537248 100644 --- a/cloudinit/tests/test_conftest.py +++ b/cloudinit/tests/test_conftest.py @@ -1,6 +1,6 @@ import pytest -from cloudinit import util +from cloudinit import subp from cloudinit.tests.helpers import CiTestCase @@ -9,17 +9,17 @@ class TestDisableSubpUsage: def test_using_subp_raises_assertion_error(self): with pytest.raises(AssertionError): - util.subp(["some", "args"]) + subp.subp(["some", "args"]) def test_typeerrors_on_incorrect_usage(self): with pytest.raises(TypeError): # We are intentionally passing no value for a parameter, so: # pylint: disable=no-value-for-parameter - util.subp() + subp.subp() @pytest.mark.parametrize('disable_subp_usage', [False], indirect=True) def test_subp_usage_can_be_reenabled(self): - util.subp(['whoami']) + subp.subp(['whoami']) @pytest.mark.parametrize( 'disable_subp_usage', [['whoami'], 'whoami'], indirect=True) @@ -27,18 +27,18 @@ class TestDisableSubpUsage: # The two parameters test each potential invocation with a single # argument with pytest.raises(AssertionError) as excinfo: - util.subp(["some", "args"]) + subp.subp(["some", "args"]) assert "allowed: whoami" in str(excinfo.value) - util.subp(['whoami']) + subp.subp(['whoami']) @pytest.mark.parametrize( 'disable_subp_usage', [['whoami', 'bash']], indirect=True) def test_subp_usage_can_be_conditionally_reenabled_for_multiple_cmds(self): with pytest.raises(AssertionError) as excinfo: - util.subp(["some", "args"]) + subp.subp(["some", "args"]) assert "allowed: whoami,bash" in str(excinfo.value) - util.subp(['bash', '-c', 'true']) - util.subp(['whoami']) + subp.subp(['bash', '-c', 'true']) + subp.subp(['whoami']) class TestDisableSubpUsageInTestSubclass(CiTestCase): @@ -46,16 +46,16 @@ class TestDisableSubpUsageInTestSubclass(CiTestCase): def test_using_subp_raises_exception(self): with pytest.raises(Exception): - util.subp(["some", "args"]) + subp.subp(["some", "args"]) def test_typeerrors_on_incorrect_usage(self): with pytest.raises(TypeError): - util.subp() + subp.subp() def test_subp_usage_can_be_reenabled(self): _old_allowed_subp = self.allow_subp self.allowed_subp = True try: - util.subp(['bash', '-c', 'true']) + subp.subp(['bash', '-c', 'true']) finally: self.allowed_subp = _old_allowed_subp diff --git a/cloudinit/tests/test_gpg.py b/cloudinit/tests/test_gpg.py index 8dd57137..f96f5372 100644 --- a/cloudinit/tests/test_gpg.py +++ b/cloudinit/tests/test_gpg.py @@ -4,19 +4,19 @@ from unittest import mock from cloudinit import gpg -from cloudinit import util +from cloudinit import subp from cloudinit.tests.helpers import CiTestCase @mock.patch("cloudinit.gpg.time.sleep") -@mock.patch("cloudinit.gpg.util.subp") +@mock.patch("cloudinit.gpg.subp.subp") class TestReceiveKeys(CiTestCase): """Test the recv_key method.""" def test_retries_on_subp_exc(self, m_subp, m_sleep): """retry should be done on gpg receive keys failure.""" retries = (1, 2, 4) - my_exc = util.ProcessExecutionError( + my_exc = subp.ProcessExecutionError( stdout='', stderr='', exit_code=2, cmd=['mycmd']) m_subp.side_effect = (my_exc, my_exc, ('', '')) gpg.recv_key("ABCD", "keyserver.example.com", retries=retries) @@ -26,7 +26,7 @@ class TestReceiveKeys(CiTestCase): """If the final run fails, error should be raised.""" naplen = 1 keyid, keyserver = ("ABCD", "keyserver.example.com") - m_subp.side_effect = util.ProcessExecutionError( + m_subp.side_effect = subp.ProcessExecutionError( stdout='', stderr='', exit_code=2, cmd=['mycmd']) with self.assertRaises(ValueError) as rcm: gpg.recv_key(keyid, keyserver, retries=(naplen,)) @@ -36,7 +36,7 @@ class TestReceiveKeys(CiTestCase): def test_no_retries_on_none(self, m_subp, m_sleep): """retry should not be done if retries is None.""" - m_subp.side_effect = util.ProcessExecutionError( + m_subp.side_effect = subp.ProcessExecutionError( stdout='', stderr='', exit_code=2, cmd=['mycmd']) with self.assertRaises(ValueError): gpg.recv_key("ABCD", "keyserver.example.com", retries=None) diff --git a/cloudinit/tests/test_netinfo.py b/cloudinit/tests/test_netinfo.py index 1c8a791e..e44b16d8 100644 --- a/cloudinit/tests/test_netinfo.py +++ b/cloudinit/tests/test_netinfo.py @@ -27,8 +27,8 @@ class TestNetInfo(CiTestCase): maxDiff = None with_logs = True - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_old_nettools_pformat(self, m_subp, m_which): """netdev_pformat properly rendering old nettools info.""" m_subp.return_value = (SAMPLE_OLD_IFCONFIG_OUT, '') @@ -36,8 +36,8 @@ class TestNetInfo(CiTestCase): content = netdev_pformat() self.assertEqual(NETDEV_FORMATTED_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_new_nettools_pformat(self, m_subp, m_which): """netdev_pformat properly rendering netdev new nettools info.""" m_subp.return_value = (SAMPLE_NEW_IFCONFIG_OUT, '') @@ -45,8 +45,8 @@ class TestNetInfo(CiTestCase): content = netdev_pformat() self.assertEqual(NETDEV_FORMATTED_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_freebsd_nettools_pformat(self, m_subp, m_which): """netdev_pformat properly rendering netdev new nettools info.""" m_subp.return_value = (SAMPLE_FREEBSD_IFCONFIG_OUT, '') @@ -57,8 +57,8 @@ class TestNetInfo(CiTestCase): print() self.assertEqual(FREEBSD_NETDEV_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_iproute_pformat(self, m_subp, m_which): """netdev_pformat properly rendering ip route info.""" m_subp.return_value = (SAMPLE_IPADDRSHOW_OUT, '') @@ -72,8 +72,8 @@ class TestNetInfo(CiTestCase): '255.0.0.0 | . |', '255.0.0.0 | host |') self.assertEqual(new_output, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_warn_on_missing_commands(self, m_subp, m_which): """netdev_pformat warns when missing both ip and 'netstat'.""" m_which.return_value = None # Niether ip nor netstat found @@ -85,8 +85,8 @@ class TestNetInfo(CiTestCase): self.logs.getvalue()) m_subp.assert_not_called() - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_info_nettools_down(self, m_subp, m_which): """test netdev_info using nettools and down interfaces.""" m_subp.return_value = ( @@ -100,8 +100,8 @@ class TestNetInfo(CiTestCase): 'hwaddr': '.', 'up': True}}, netdev_info(".")) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_netdev_info_iproute_down(self, m_subp, m_which): """Test netdev_info with ip and down interfaces.""" m_subp.return_value = ( @@ -130,8 +130,8 @@ class TestNetInfo(CiTestCase): readResource("netinfo/netdev-formatted-output-down"), netdev_pformat()) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_route_nettools_pformat(self, m_subp, m_which): """route_pformat properly rendering nettools route info.""" @@ -147,8 +147,8 @@ class TestNetInfo(CiTestCase): content = route_pformat() self.assertEqual(ROUTE_FORMATTED_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_route_iproute_pformat(self, m_subp, m_which): """route_pformat properly rendering ip route info.""" @@ -165,8 +165,8 @@ class TestNetInfo(CiTestCase): content = route_pformat() self.assertEqual(ROUTE_FORMATTED_OUT, content) - @mock.patch('cloudinit.netinfo.util.which') - @mock.patch('cloudinit.netinfo.util.subp') + @mock.patch('cloudinit.netinfo.subp.which') + @mock.patch('cloudinit.netinfo.subp.subp') def test_route_warn_on_missing_commands(self, m_subp, m_which): """route_pformat warns when missing both ip and 'netstat'.""" m_which.return_value = None # Niether ip nor netstat found diff --git a/cloudinit/tests/test_subp.py b/cloudinit/tests/test_subp.py index 448097d3..911c1f3d 100644 --- a/cloudinit/tests/test_subp.py +++ b/cloudinit/tests/test_subp.py @@ -2,10 +2,21 @@ """Tests for cloudinit.subp utility functions""" -from cloudinit import subp +import json +import os +import sys +import stat + +from unittest import mock + +from cloudinit import subp, util from cloudinit.tests.helpers import CiTestCase +BASH = subp.which('bash') +BOGUS_COMMAND = 'this-is-not-expected-to-be-a-program-name' + + class TestPrependBaseCommands(CiTestCase): with_logs = True @@ -58,4 +69,218 @@ class TestPrependBaseCommands(CiTestCase): self.assertEqual('', self.logs.getvalue()) self.assertEqual(expected, fixed_commands) + +class TestSubp(CiTestCase): + allowed_subp = [BASH, 'cat', CiTestCase.SUBP_SHELL_TRUE, + BOGUS_COMMAND, sys.executable] + + stdin2err = [BASH, '-c', 'cat >&2'] + stdin2out = ['cat'] + utf8_invalid = b'ab\xaadef' + utf8_valid = b'start \xc3\xa9 end' + utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7' + printenv = [BASH, '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--'] + + def printf_cmd(self, *args): + # bash's printf supports \xaa. So does /usr/bin/printf + # but by using bash, we remove dependency on another program. + return([BASH, '-c', 'printf "$@"', 'printf'] + list(args)) + + def test_subp_handles_bytestrings(self): + """subp can run a bytestring command if shell is True.""" + tmp_file = self.tmp_path('test.out') + cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file) + (out, _err) = subp.subp(cmd.encode('utf-8'), shell=True) + self.assertEqual(u'', out) + self.assertEqual(u'', _err) + self.assertEqual('HI MOM\n', util.load_file(tmp_file)) + + def test_subp_handles_strings(self): + """subp can run a string command if shell is True.""" + tmp_file = self.tmp_path('test.out') + cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file) + (out, _err) = subp.subp(cmd, shell=True) + self.assertEqual(u'', out) + self.assertEqual(u'', _err) + self.assertEqual('HI MOM\n', util.load_file(tmp_file)) + + def test_subp_handles_utf8(self): + # The given bytes contain utf-8 accented characters as seen in e.g. + # the "deja dup" package in Ubuntu. + cmd = self.printf_cmd(self.utf8_valid_2) + (out, _err) = subp.subp(cmd, capture=True) + self.assertEqual(out, self.utf8_valid_2.decode('utf-8')) + + def test_subp_respects_decode_false(self): + (out, err) = subp.subp(self.stdin2out, capture=True, decode=False, + data=self.utf8_valid) + self.assertTrue(isinstance(out, bytes)) + self.assertTrue(isinstance(err, bytes)) + self.assertEqual(out, self.utf8_valid) + + def test_subp_decode_ignore(self): + # this executes a string that writes invalid utf-8 to stdout + (out, _err) = subp.subp(self.printf_cmd('abc\\xaadef'), + capture=True, decode='ignore') + self.assertEqual(out, 'abcdef') + + def test_subp_decode_strict_valid_utf8(self): + (out, _err) = subp.subp(self.stdin2out, capture=True, + decode='strict', data=self.utf8_valid) + self.assertEqual(out, self.utf8_valid.decode('utf-8')) + + def test_subp_decode_invalid_utf8_replaces(self): + (out, _err) = subp.subp(self.stdin2out, capture=True, + data=self.utf8_invalid) + expected = self.utf8_invalid.decode('utf-8', 'replace') + self.assertEqual(out, expected) + + def test_subp_decode_strict_raises(self): + args = [] + kwargs = {'args': self.stdin2out, 'capture': True, + 'decode': 'strict', 'data': self.utf8_invalid} + self.assertRaises(UnicodeDecodeError, subp.subp, *args, **kwargs) + + def test_subp_capture_stderr(self): + data = b'hello world' + (out, err) = subp.subp(self.stdin2err, capture=True, + decode=False, data=data, + update_env={'LC_ALL': 'C'}) + self.assertEqual(err, data) + self.assertEqual(out, b'') + + def test_subp_reads_env(self): + with mock.patch.dict("os.environ", values={'FOO': 'BAR'}): + out, _err = subp.subp(self.printenv + ['FOO'], capture=True) + self.assertEqual('FOO=BAR', out.splitlines()[0]) + + def test_subp_env_and_update_env(self): + out, _err = subp.subp( + self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True, + env={'FOO': 'BAR'}, + update_env={'HOME': '/myhome', 'K2': 'V2'}) + self.assertEqual( + ['FOO=BAR', 'HOME=/myhome', 'K1=', 'K2=V2'], out.splitlines()) + + def test_subp_update_env(self): + extra = {'FOO': 'BAR', 'HOME': '/root', 'K1': 'V1'} + with mock.patch.dict("os.environ", values=extra): + out, _err = subp.subp( + self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True, + update_env={'HOME': '/myhome', 'K2': 'V2'}) + + self.assertEqual( + ['FOO=BAR', 'HOME=/myhome', 'K1=V1', 'K2=V2'], out.splitlines()) + + def test_subp_warn_missing_shebang(self): + """Warn on no #! in script""" + noshebang = self.tmp_path('noshebang') + util.write_file(noshebang, 'true\n') + + print("os is %s" % os) + os.chmod(noshebang, os.stat(noshebang).st_mode | stat.S_IEXEC) + with self.allow_subp([noshebang]): + self.assertRaisesRegex(subp.ProcessExecutionError, + r'Missing #! in script\?', + subp.subp, (noshebang,)) + + def test_subp_combined_stderr_stdout(self): + """Providing combine_capture as True redirects stderr to stdout.""" + data = b'hello world' + (out, err) = subp.subp(self.stdin2err, capture=True, + combine_capture=True, decode=False, data=data) + self.assertEqual(b'', err) + self.assertEqual(data, out) + + def test_returns_none_if_no_capture(self): + (out, err) = subp.subp(self.stdin2out, data=b'', capture=False) + self.assertIsNone(err) + self.assertIsNone(out) + + def test_exception_has_out_err_are_bytes_if_decode_false(self): + """Raised exc should have stderr, stdout as bytes if no decode.""" + with self.assertRaises(subp.ProcessExecutionError) as cm: + subp.subp([BOGUS_COMMAND], decode=False) + self.assertTrue(isinstance(cm.exception.stdout, bytes)) + self.assertTrue(isinstance(cm.exception.stderr, bytes)) + + def test_exception_has_out_err_are_bytes_if_decode_true(self): + """Raised exc should have stderr, stdout as string if no decode.""" + with self.assertRaises(subp.ProcessExecutionError) as cm: + subp.subp([BOGUS_COMMAND], decode=True) + self.assertTrue(isinstance(cm.exception.stdout, str)) + self.assertTrue(isinstance(cm.exception.stderr, str)) + + def test_bunch_of_slashes_in_path(self): + self.assertEqual("/target/my/path/", + subp.target_path("/target/", "//my/path/")) + self.assertEqual("/target/my/path/", + subp.target_path("/target/", "///my/path/")) + + def test_c_lang_can_take_utf8_args(self): + """Independent of system LC_CTYPE, args can contain utf-8 strings. + + When python starts up, its default encoding gets set based on + the value of LC_CTYPE. If no system locale is set, the default + encoding for both python2 and python3 in some paths will end up + being ascii. + + Attempts to use setlocale or patching (or changing) os.environ + in the current environment seem to not be effective. + + This test starts up a python with LC_CTYPE set to C so that + the default encoding will be set to ascii. In such an environment + Popen(['command', 'non-ascii-arg']) would cause a UnicodeDecodeError. + """ + python_prog = '\n'.join([ + 'import json, sys', + 'from cloudinit.subp import subp', + 'data = sys.stdin.read()', + 'cmd = json.loads(data)', + 'subp(cmd, capture=False)', + '']) + cmd = [BASH, '-c', 'echo -n "$@"', '--', + self.utf8_valid.decode("utf-8")] + python_subp = [sys.executable, '-c', python_prog] + + out, _err = subp.subp( + python_subp, update_env={'LC_CTYPE': 'C'}, + data=json.dumps(cmd).encode("utf-8"), + decode=False) + self.assertEqual(self.utf8_valid, out) + + def test_bogus_command_logs_status_messages(self): + """status_cb gets status messages logs on bogus commands provided.""" + logs = [] + + def status_cb(log): + logs.append(log) + + with self.assertRaises(subp.ProcessExecutionError): + subp.subp([BOGUS_COMMAND], status_cb=status_cb) + + expected = [ + 'Begin run command: {cmd}\n'.format(cmd=BOGUS_COMMAND), + 'ERROR: End run command: invalid command provided\n'] + self.assertEqual(expected, logs) + + def test_command_logs_exit_codes_to_status_cb(self): + """status_cb gets status messages containing command exit code.""" + logs = [] + + def status_cb(log): + logs.append(log) + + with self.assertRaises(subp.ProcessExecutionError): + subp.subp([BASH, '-c', 'exit 2'], status_cb=status_cb) + subp.subp([BASH, '-c', 'exit 0'], status_cb=status_cb) + + expected = [ + 'Begin run command: %s -c exit 2\n' % BASH, + 'ERROR: End run command: exit(2)\n', + 'Begin run command: %s -c exit 0\n' % BASH, + 'End run command: exit(0)\n'] + self.assertEqual(expected, logs) + + # vi: ts=4 expandtab diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index bfccfe1e..0d9a8fd9 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -9,6 +9,7 @@ import platform import pytest import cloudinit.util as util +from cloudinit import subp from cloudinit.tests.helpers import CiTestCase, mock from textwrap import dedent @@ -332,7 +333,7 @@ class TestBlkid(CiTestCase): "PARTUUID": self.ids["id09"]}, }) - @mock.patch("cloudinit.util.subp") + @mock.patch("cloudinit.subp.subp") def test_functional_blkid(self, m_subp): m_subp.return_value = ( self.blkid_out.format(**self.ids), "") @@ -340,7 +341,7 @@ class TestBlkid(CiTestCase): m_subp.assert_called_with(["blkid", "-o", "full"], capture=True, decode="replace") - @mock.patch("cloudinit.util.subp") + @mock.patch("cloudinit.subp.subp") def test_blkid_no_cache_uses_no_cache(self, m_subp): """blkid should turn off cache if disable_cache is true.""" m_subp.return_value = ( @@ -351,7 +352,7 @@ class TestBlkid(CiTestCase): capture=True, decode="replace") -@mock.patch('cloudinit.util.subp') +@mock.patch('cloudinit.subp.subp') class TestUdevadmSettle(CiTestCase): def test_with_no_params(self, m_subp): """called with no parameters.""" @@ -396,8 +397,8 @@ class TestUdevadmSettle(CiTestCase): '--timeout=%s' % timeout]) def test_subp_exception_raises_to_caller(self, m_subp): - m_subp.side_effect = util.ProcessExecutionError("BOOM") - self.assertRaises(util.ProcessExecutionError, util.udevadm_settle) + m_subp.side_effect = subp.ProcessExecutionError("BOOM") + self.assertRaises(subp.ProcessExecutionError, util.udevadm_settle) @mock.patch('os.path.exists') diff --git a/cloudinit/util.py b/cloudinit/util.py index 985e7d20..445e3d4c 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -32,12 +32,13 @@ import subprocess import sys import time from base64 import b64decode, b64encode -from errno import ENOENT, ENOEXEC +from errno import ENOENT from functools import lru_cache from urllib import parse from cloudinit import importer from cloudinit import log as logging +from cloudinit import subp from cloudinit import ( mergers, safeyaml, @@ -74,8 +75,8 @@ def get_dpkg_architecture(target=None): N.B. This function is wrapped in functools.lru_cache, so repeated calls won't shell out every time. """ - out, _ = subp(['dpkg', '--print-architecture'], capture=True, - target=target) + out, _ = subp.subp(['dpkg', '--print-architecture'], capture=True, + target=target) return out.strip() @@ -86,7 +87,8 @@ def lsb_release(target=None): data = {} try: - out, _ = subp(['lsb_release', '--all'], capture=True, target=target) + out, _ = subp.subp(['lsb_release', '--all'], capture=True, + target=target) for line in out.splitlines(): fname, _, val = line.partition(":") if fname in fmap: @@ -96,35 +98,13 @@ def lsb_release(target=None): LOG.warning("Missing fields in lsb_release --all output: %s", ','.join(missing)) - except ProcessExecutionError as err: + except subp.ProcessExecutionError as err: LOG.warning("Unable to get lsb_release --all: %s", err) data = dict((v, "UNAVAILABLE") for v in fmap.values()) return data -def target_path(target, path=None): - # return 'path' inside target, accepting target as None - if target in (None, ""): - target = "/" - elif not isinstance(target, str): - raise ValueError("Unexpected input for target: %s" % target) - else: - target = os.path.abspath(target) - # abspath("//") returns "//" specifically for 2 slashes. - if target.startswith("//"): - target = target[1:] - - if not path: - return target - - # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /. - while len(path) and path[0] == "/": - path = path[1:] - - return os.path.join(target, path) - - def decode_binary(blob, encoding='utf-8'): # Converts a binary type into a text type using given encoding. if isinstance(blob, str): @@ -201,91 +181,6 @@ DMIDECODE_TO_DMI_SYS_MAPPING = { } -class ProcessExecutionError(IOError): - - MESSAGE_TMPL = ('%(description)s\n' - 'Command: %(cmd)s\n' - 'Exit code: %(exit_code)s\n' - 'Reason: %(reason)s\n' - 'Stdout: %(stdout)s\n' - 'Stderr: %(stderr)s') - empty_attr = '-' - - def __init__(self, stdout=None, stderr=None, - exit_code=None, cmd=None, - description=None, reason=None, - errno=None): - if not cmd: - self.cmd = self.empty_attr - else: - self.cmd = cmd - - if not description: - if not exit_code and errno == ENOEXEC: - self.description = 'Exec format error. Missing #! in script?' - else: - self.description = 'Unexpected error while running command.' - else: - self.description = description - - if not isinstance(exit_code, int): - self.exit_code = self.empty_attr - else: - self.exit_code = exit_code - - if not stderr: - if stderr is None: - self.stderr = self.empty_attr - else: - self.stderr = stderr - else: - self.stderr = self._indent_text(stderr) - - if not stdout: - if stdout is None: - self.stdout = self.empty_attr - else: - self.stdout = stdout - else: - self.stdout = self._indent_text(stdout) - - if reason: - self.reason = reason - else: - self.reason = self.empty_attr - - self.errno = errno - message = self.MESSAGE_TMPL % { - 'description': self._ensure_string(self.description), - 'cmd': self._ensure_string(self.cmd), - 'exit_code': self._ensure_string(self.exit_code), - 'stdout': self._ensure_string(self.stdout), - 'stderr': self._ensure_string(self.stderr), - 'reason': self._ensure_string(self.reason), - } - IOError.__init__(self, message) - - def _ensure_string(self, text): - """ - if data is bytes object, decode - """ - return text.decode() if isinstance(text, bytes) else text - - def _indent_text(self, text, indent_level=8): - """ - indent text on all but the first line, allowing for easy to read output - """ - cr = '\n' - indent = ' ' * indent_level - # if input is bytes, return bytes - if isinstance(text, bytes): - cr = cr.encode() - indent = indent.encode() - # remove any newlines at end of text first to prevent unneeded blank - # line in output - return text.rstrip(cr).replace(cr, cr + indent) - - class SeLinuxGuard(object): def __init__(self, path, recursive=False): # Late import since it might not always @@ -875,8 +770,8 @@ def runparts(dirp, skip_no_exist=True, exe_prefix=None): if os.path.isfile(exe_path) and os.access(exe_path, os.X_OK): attempted.append(exe_path) try: - subp(prefix + [exe_path], capture=False) - except ProcessExecutionError as e: + subp.subp(prefix + [exe_path], capture=False) + except subp.ProcessExecutionError as e: logexc(LOG, "Failed running %s [%s]", exe_path, e.exit_code) failed.append(e) @@ -1271,10 +1166,10 @@ def find_devs_with_netbsd(criteria=None, oformat='device', label = criteria.lstrip("LABEL=") if criteria.startswith("TYPE="): _type = criteria.lstrip("TYPE=") - out, _err = subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) + out, _err = subp.subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) for dev in out.split(): if label or _type: - mscdlabel_out, _ = subp(['mscdlabel', dev], rcs=[0, 1]) + mscdlabel_out, _ = subp.subp(['mscdlabel', dev], rcs=[0, 1]) if label and not ('label "%s"' % label) in mscdlabel_out: continue if _type == "iso9660" and "ISO filesystem" not in mscdlabel_out: @@ -1287,7 +1182,7 @@ def find_devs_with_netbsd(criteria=None, oformat='device', def find_devs_with_openbsd(criteria=None, oformat='device', tag=None, no_cache=False, path=None): - out, _err = subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) + out, _err = subp.subp(['sysctl', '-n', 'hw.disknames'], rcs=[0]) devlist = [] for entry in out.split(','): if not entry.endswith(':'): @@ -1353,8 +1248,8 @@ def find_devs_with(criteria=None, oformat='device', cmd = blk_id_cmd + options # See man blkid for why 2 is added try: - (out, _err) = subp(cmd, rcs=[0, 2]) - except ProcessExecutionError as e: + (out, _err) = subp.subp(cmd, rcs=[0, 2]) + except subp.ProcessExecutionError as e: if e.errno == ENOENT: # blkid not found... out = "" @@ -1389,7 +1284,7 @@ def blkid(devs=None, disable_cache=False): # we have to decode with 'replace' as shelx.split (called by # load_shell_content) can't take bytes. So this is potentially # lossy of non-utf-8 chars in blkid output. - out, _ = subp(cmd, capture=True, decode="replace") + out, _ = subp.subp(cmd, capture=True, decode="replace") ret = {} for line in out.splitlines(): dev, _, data = line.partition(":") @@ -1709,7 +1604,7 @@ def unmounter(umount): finally: if umount: umount_cmd = ["umount", umount] - subp(umount_cmd) + subp.subp(umount_cmd) def mounts(): @@ -1720,7 +1615,7 @@ def mounts(): mount_locs = load_file("/proc/mounts").splitlines() method = 'proc' else: - (mountoutput, _err) = subp("mount") + (mountoutput, _err) = subp.subp("mount") mount_locs = mountoutput.splitlines() method = 'mount' mountre = r'^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$' @@ -1804,7 +1699,7 @@ def mount_cb(device, callback, data=None, mtype=None, mountcmd.extend(['-t', mtype]) mountcmd.append(device) mountcmd.append(tmpd) - subp(mountcmd, update_env=update_env_for_mount) + subp.subp(mountcmd, update_env=update_env_for_mount) umount = tmpd # This forces it to be unmounted (when set) mountpoint = tmpd break @@ -1988,185 +1883,6 @@ def delete_dir_contents(dirname): del_file(node_fullpath) -def subp_blob_in_tempfile(blob, *args, **kwargs): - """Write blob to a tempfile, and call subp with args, kwargs. Then cleanup. - - 'basename' as a kwarg allows providing the basename for the file. - The 'args' argument to subp will be updated with the full path to the - filename as the first argument. - """ - basename = kwargs.pop('basename', "subp_blob") - - if len(args) == 0 and 'args' not in kwargs: - args = [tuple()] - - # Use tmpdir over tmpfile to avoid 'text file busy' on execute - with temp_utils.tempdir(needs_exe=True) as tmpd: - tmpf = os.path.join(tmpd, basename) - if 'args' in kwargs: - kwargs['args'] = [tmpf] + list(kwargs['args']) - else: - args = list(args) - args[0] = [tmpf] + args[0] - - write_file(tmpf, blob, mode=0o700) - return subp(*args, **kwargs) - - -def subp(args, data=None, rcs=None, env=None, capture=True, - combine_capture=False, shell=False, - logstring=False, decode="replace", target=None, update_env=None, - status_cb=None): - """Run a subprocess. - - :param args: command to run in a list. [cmd, arg1, arg2...] - :param data: input to the command, made available on its stdin. - :param rcs: - a list of allowed return codes. If subprocess exits with a value not - in this list, a ProcessExecutionError will be raised. By default, - data is returned as a string. See 'decode' parameter. - :param env: a dictionary for the command's environment. - :param capture: - boolean indicating if output should be captured. If True, then stderr - and stdout will be returned. If False, they will not be redirected. - :param combine_capture: - boolean indicating if stderr should be redirected to stdout. When True, - interleaved stderr and stdout will be returned as the first element of - a tuple, the second will be empty string or bytes (per decode). - if combine_capture is True, then output is captured independent of - the value of capture. - :param shell: boolean indicating if this should be run with a shell. - :param logstring: - the command will be logged to DEBUG. If it contains info that should - not be logged, then logstring will be logged instead. - :param decode: - if False, no decoding will be done and returned stdout and stderr will - be bytes. Other allowed values are 'strict', 'ignore', and 'replace'. - These values are passed through to bytes().decode() as the 'errors' - parameter. There is no support for decoding to other than utf-8. - :param target: - not supported, kwarg present only to make function signature similar - to curtin's subp. - :param update_env: - update the enviornment for this command with this dictionary. - this will not affect the current processes os.environ. - :param status_cb: - call this fuction with a single string argument before starting - and after finishing. - - :return - if not capturing, return is (None, None) - if capturing, stdout and stderr are returned. - if decode: - entries in tuple will be python2 unicode or python3 string - if not decode: - entries in tuple will be python2 string or python3 bytes - """ - - # not supported in cloud-init (yet), for now kept in the call signature - # to ease maintaining code shared between cloud-init and curtin - if target is not None: - raise ValueError("target arg not supported by cloud-init") - - if rcs is None: - rcs = [0] - - devnull_fp = None - - if update_env: - if env is None: - env = os.environ - env = env.copy() - env.update(update_env) - - if target_path(target) != "/": - args = ['chroot', target] + list(args) - - if status_cb: - command = ' '.join(args) if isinstance(args, list) else args - status_cb('Begin run command: {command}\n'.format(command=command)) - if not logstring: - LOG.debug(("Running command %s with allowed return codes %s" - " (shell=%s, capture=%s)"), - args, rcs, shell, 'combine' if combine_capture else capture) - else: - LOG.debug(("Running hidden command to protect sensitive " - "input/output logstring: %s"), logstring) - - stdin = None - stdout = None - stderr = None - if capture: - stdout = subprocess.PIPE - stderr = subprocess.PIPE - if combine_capture: - stdout = subprocess.PIPE - stderr = subprocess.STDOUT - if data is None: - # using devnull assures any reads get null, rather - # than possibly waiting on input. - devnull_fp = open(os.devnull) - stdin = devnull_fp - else: - stdin = subprocess.PIPE - if not isinstance(data, bytes): - data = data.encode() - - # Popen converts entries in the arguments array from non-bytes to bytes. - # When locale is unset it may use ascii for that encoding which can - # cause UnicodeDecodeErrors. (LP: #1751051) - if isinstance(args, bytes): - bytes_args = args - elif isinstance(args, str): - bytes_args = args.encode("utf-8") - else: - bytes_args = [ - x if isinstance(x, bytes) else x.encode("utf-8") - for x in args] - try: - sp = subprocess.Popen(bytes_args, stdout=stdout, - stderr=stderr, stdin=stdin, - env=env, shell=shell) - (out, err) = sp.communicate(data) - except OSError as e: - if status_cb: - status_cb('ERROR: End run command: invalid command provided\n') - raise ProcessExecutionError( - cmd=args, reason=e, errno=e.errno, - stdout="-" if decode else b"-", - stderr="-" if decode else b"-") - finally: - if devnull_fp: - devnull_fp.close() - - # Just ensure blank instead of none. - if capture or combine_capture: - if not out: - out = b'' - if not err: - err = b'' - if decode: - def ldecode(data, m='utf-8'): - if not isinstance(data, bytes): - return data - return data.decode(m, decode) - - out = ldecode(out) - err = ldecode(err) - - rc = sp.returncode - if rc not in rcs: - if status_cb: - status_cb( - 'ERROR: End run command: exit({code})\n'.format(code=rc)) - raise ProcessExecutionError(stdout=out, stderr=err, - exit_code=rc, - cmd=args) - if status_cb: - status_cb('End run command: exit({code})\n'.format(code=rc)) - return (out, err) - - def make_header(comment_char="#", base='created'): ci_ver = version.version_string() header = str(comment_char) @@ -2232,7 +1948,7 @@ def is_container(): try: # try to run a helper program. if it returns true/zero # then we're inside a container. otherwise, no - subp(helper) + subp.subp(helper) return True except (IOError, OSError): pass @@ -2438,7 +2154,7 @@ def find_freebsd_part(fs): return splitted[2] elif splitted[2] in ['label', 'gpt', 'ufs']: target_label = fs[5:] - (part, _err) = subp(['glabel', 'status', '-s']) + (part, _err) = subp.subp(['glabel', 'status', '-s']) for labels in part.split("\n"): items = labels.split() if len(items) > 0 and items[0] == target_label: @@ -2460,10 +2176,10 @@ def get_path_dev_freebsd(path, mnt_list): def get_mount_info_freebsd(path): - (result, err) = subp(['mount', '-p', path], rcs=[0, 1]) + (result, err) = subp.subp(['mount', '-p', path], rcs=[0, 1]) if len(err): # find a path if the input is not a mounting point - (mnt_list, err) = subp(['mount', '-p']) + (mnt_list, err) = subp.subp(['mount', '-p']) path_found = get_path_dev_freebsd(path, mnt_list) if (path_found is None): return None @@ -2479,8 +2195,8 @@ def get_device_info_from_zpool(zpool): LOG.debug('Cannot get zpool info, no /dev/zfs') return None try: - (zpoolstatus, err) = subp(['zpool', 'status', zpool]) - except ProcessExecutionError as err: + (zpoolstatus, err) = subp.subp(['zpool', 'status', zpool]) + except subp.ProcessExecutionError as err: LOG.warning("Unable to get zpool status of %s: %s", zpool, err) return None if len(err): @@ -2494,7 +2210,7 @@ def get_device_info_from_zpool(zpool): def parse_mount(path): - (mountoutput, _err) = subp(['mount']) + (mountoutput, _err) = subp.subp(['mount']) mount_locs = mountoutput.splitlines() # there are 2 types of mount outputs we have to parse therefore # the regex is a bit complex. to better understand this regex see: @@ -2567,40 +2283,6 @@ def get_mount_info(path, log=LOG, get_mnt_opts=False): return parse_mount(path) -def is_exe(fpath): - # return boolean indicating if fpath exists and is executable. - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - -def which(program, search=None, target=None): - target = target_path(target) - - if os.path.sep in program: - # if program had a '/' in it, then do not search PATH - # 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls - # so effectively we set cwd to / (or target) - if is_exe(target_path(target, program)): - return program - - if search is None: - paths = [p.strip('"') for p in - os.environ.get("PATH", "").split(os.pathsep)] - if target == "/": - search = paths - else: - search = [p for p in paths if p.startswith("/")] - - # normalize path input - search = [os.path.abspath(p) for p in search] - - for path in search: - ppath = os.path.sep.join((path, program)) - if is_exe(target_path(target, ppath)): - return ppath - - return None - - def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): if args is None: args = [] @@ -2764,7 +2446,7 @@ def _call_dmidecode(key, dmidecode_path): """ try: cmd = [dmidecode_path, "--string", key] - (result, _err) = subp(cmd) + (result, _err) = subp.subp(cmd) result = result.strip() LOG.debug("dmidecode returned '%s' for '%s'", result, key) if result.replace(".", "") == "": @@ -2818,7 +2500,8 @@ def read_dmi_data(key): LOG.debug("dmidata is not supported on %s", uname_arch) return None - dmidecode_path = which('dmidecode') + print("hi, now its: %s\n", subp) + dmidecode_path = subp.which('dmidecode') if dmidecode_path: return _call_dmidecode(key, dmidecode_path) @@ -2834,7 +2517,7 @@ def message_from_string(string): def get_installed_packages(target=None): - (out, _) = subp(['dpkg-query', '--list'], target=target, capture=True) + (out, _) = subp.subp(['dpkg-query', '--list'], target=target, capture=True) pkgs_inst = set() for line in out.splitlines(): @@ -2970,7 +2653,7 @@ def udevadm_settle(exists=None, timeout=None): if timeout: settle_cmd.extend(['--timeout=%s' % timeout]) - return subp(settle_cmd) + return subp.subp(settle_cmd) def get_proc_ppid(pid): diff --git a/packages/bddeb b/packages/bddeb index 02ac2975..78b1c83b 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -24,6 +24,7 @@ def find_root(): if "avoid-pep8-E402-import-not-top-of-file": # Use the util functions from cloudinit sys.path.insert(0, find_root()) + from cloudinit import subp from cloudinit import util from cloudinit import temp_utils from cloudinit import templater @@ -53,7 +54,7 @@ def run_helper(helper, args=None, strip=True): if args is None: args = [] cmd = [util.abs_join(find_root(), 'tools', helper)] + args - (stdout, _stderr) = util.subp(cmd) + (stdout, _stderr) = subp.subp(cmd) if strip: stdout = stdout.strip() return stdout @@ -67,7 +68,7 @@ def write_debian_folder(root, templ_data, cloud_util_deps): # Just copy debian/ dir and then update files pdeb_d = util.abs_join(find_root(), 'packages', 'debian') - util.subp(['cp', '-a', pdeb_d, deb_dir]) + subp.subp(['cp', '-a', pdeb_d, deb_dir]) # Fill in the change log template templater.render_to_file(util.abs_join(find_root(), @@ -190,7 +191,7 @@ def main(): print("Extracting temporary tarball %r" % (tarball)) cmd = ['tar', '-xvzf', tarball_fp, '-C', tdir] - util.subp(cmd, capture=capture) + subp.subp(cmd, capture=capture) xdir = util.abs_join(tdir, "cloud-init-%s" % ver_data['version_long']) templ_data.update(ver_data) @@ -203,7 +204,7 @@ def main(): cmd = ['debuild', '--preserve-envvar', 'INIT_SYSTEM'] if args.debuild_args: cmd.extend(args.debuild_args) - util.subp(cmd, capture=capture) + subp.subp(cmd, capture=capture) link_fn = os.path.join(os.getcwd(), 'cloud-init_all.deb') link_dsc = os.path.join(os.getcwd(), 'cloud-init.dsc') diff --git a/packages/brpm b/packages/brpm index 1be8804c..56477d2e 100755 --- a/packages/brpm +++ b/packages/brpm @@ -24,6 +24,7 @@ def find_root(): if "avoid-pep8-E402-import-not-top-of-file": # Use the util functions from cloudinit sys.path.insert(0, find_root()) + from cloudinit import subp from cloudinit import templater from cloudinit import util @@ -36,7 +37,7 @@ def run_helper(helper, args=None, strip=True): if args is None: args = [] cmd = [util.abs_join(find_root(), 'tools', helper)] + args - (stdout, _stderr) = util.subp(cmd) + (stdout, _stderr) = subp.subp(cmd) if strip: stdout = stdout.strip() return stdout @@ -174,7 +175,7 @@ def main(): else: cmd = ['rpmbuild', '-ba', spec_fn] - util.subp(cmd, capture=capture) + subp.subp(cmd, capture=capture) # Copy the items built to our local dir globs = [] diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py index f04d0cd4..6f74436d 100644 --- a/tests/cloud_tests/bddeb.py +++ b/tests/cloud_tests/bddeb.py @@ -6,7 +6,7 @@ from functools import partial import os import tempfile -from cloudinit import util as c_util +from cloudinit import subp from tests.cloud_tests import (config, LOG) from tests.cloud_tests import platforms from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single) @@ -42,8 +42,8 @@ def build_deb(args, instance): 'GIT_WORK_TREE': extract_dir} LOG.debug('creating tarball of cloud-init at: %s', local_tarball) - c_util.subp(['tar', 'cf', local_tarball, '--owner', 'root', - '--group', 'root', '-C', args.cloud_init, '.']) + subp.subp(['tar', 'cf', local_tarball, '--owner', 'root', + '--group', 'root', '-C', args.cloud_init, '.']) LOG.debug('copying to remote system at: %s', remote_tarball) instance.push_file(local_tarball, remote_tarball) diff --git a/tests/cloud_tests/platforms/lxd/image.py b/tests/cloud_tests/platforms/lxd/image.py index b5de1f52..8934fb74 100644 --- a/tests/cloud_tests/platforms/lxd/image.py +++ b/tests/cloud_tests/platforms/lxd/image.py @@ -8,6 +8,7 @@ import tempfile from ..images import Image from .snapshot import LXDSnapshot +from cloudinit import subp from cloudinit import util as c_util from tests.cloud_tests import util @@ -81,8 +82,8 @@ class LXDImage(Image): @return_value: tuple of path to metadata tarball and rootfs tarball """ # pylxd's image export feature doesn't do split exports, so use cmdline - c_util.subp(['lxc', 'image', 'export', self.pylxd_image.fingerprint, - output_dir], capture=True) + subp.subp(['lxc', 'image', 'export', self.pylxd_image.fingerprint, + output_dir], capture=True) tarballs = [p for p in os.listdir(output_dir) if p.endswith('tar.xz')] metadata = os.path.join( output_dir, next(p for p in tarballs if p.startswith('meta-'))) @@ -101,8 +102,8 @@ class LXDImage(Image): """ alias = util.gen_instance_name( image_desc=str(self), use_desc='update-metadata') - c_util.subp(['lxc', 'image', 'import', metadata, rootfs, - '--alias', alias], capture=True) + subp.subp(['lxc', 'image', 'import', metadata, rootfs, + '--alias', alias], capture=True) self.pylxd_image = self.platform.query_image_by_alias(alias) return self.pylxd_image.fingerprint diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py index 2b804a62..b27b9848 100644 --- a/tests/cloud_tests/platforms/lxd/instance.py +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -7,7 +7,8 @@ import shutil import time from tempfile import mkdtemp -from cloudinit.util import load_yaml, subp, ProcessExecutionError, which +from cloudinit.subp import subp, ProcessExecutionError, which +from cloudinit.util import load_yaml from tests.cloud_tests import LOG from tests.cloud_tests.util import PlatformError diff --git a/tests/cloud_tests/platforms/nocloudkvm/image.py b/tests/cloud_tests/platforms/nocloudkvm/image.py index bc2b6e75..ff5b6ad7 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/image.py +++ b/tests/cloud_tests/platforms/nocloudkvm/image.py @@ -2,7 +2,7 @@ """NoCloud KVM Image Base Class.""" -from cloudinit import util as c_util +from cloudinit import subp import os import shutil @@ -30,8 +30,8 @@ class NoCloudKVMImage(Image): self._img_path = os.path.join(self._workd, os.path.basename(self._orig_img_path)) - c_util.subp(['qemu-img', 'create', '-f', 'qcow2', - '-b', orig_img_path, self._img_path]) + subp.subp(['qemu-img', 'create', '-f', 'qcow2', + '-b', orig_img_path, self._img_path]) super(NoCloudKVMImage, self).__init__(platform, config) @@ -50,10 +50,10 @@ class NoCloudKVMImage(Image): '--system-resolvconf', self._img_path, '--', 'chroot', '_MOUNTPOINT_'] try: - out, err = c_util.subp(mic_chroot + env_args + list(command), - data=stdin, decode=False) + out, err = subp.subp(mic_chroot + env_args + list(command), + data=stdin, decode=False) return (out, err, 0) - except c_util.ProcessExecutionError as e: + except subp.ProcessExecutionError as e: return (e.stdout, e.stderr, e.exit_code) def snapshot(self): diff --git a/tests/cloud_tests/platforms/nocloudkvm/instance.py b/tests/cloud_tests/platforms/nocloudkvm/instance.py index 96185b75..5140a11c 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/instance.py +++ b/tests/cloud_tests/platforms/nocloudkvm/instance.py @@ -11,7 +11,7 @@ import uuid from ..instances import Instance from cloudinit.atomic_helper import write_json -from cloudinit import util as c_util +from cloudinit import subp from tests.cloud_tests import LOG, util # This domain contains reverse lookups for hostnames that are used. @@ -110,8 +110,8 @@ class NoCloudKVMInstance(Instance): """Clean up instance.""" if self.pid: try: - c_util.subp(['kill', '-9', self.pid]) - except c_util.ProcessExecutionError: + subp.subp(['kill', '-9', self.pid]) + except subp.ProcessExecutionError: pass if self.pid_file: @@ -143,8 +143,8 @@ class NoCloudKVMInstance(Instance): # meta-data can be yaml, but more easily pretty printed with json write_json(meta_data_file, self.meta_data) - c_util.subp(['cloud-localds', seed_file, user_data_file, - meta_data_file]) + subp.subp(['cloud-localds', seed_file, user_data_file, + meta_data_file]) return seed_file diff --git a/tests/cloud_tests/platforms/nocloudkvm/platform.py b/tests/cloud_tests/platforms/nocloudkvm/platform.py index 2d1480f5..53c8ebf2 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/platform.py +++ b/tests/cloud_tests/platforms/nocloudkvm/platform.py @@ -12,6 +12,7 @@ from simplestreams import util as s_util from ..platforms import Platform from .image import NoCloudKVMImage from .instance import NoCloudKVMInstance +from cloudinit import subp from cloudinit import util as c_util from tests.cloud_tests import util @@ -84,8 +85,8 @@ class NoCloudKVMPlatform(Platform): """ name = util.gen_instance_name(image_desc=image_desc, use_desc=use_desc) img_path = os.path.join(self.config['data_dir'], name + '.qcow2') - c_util.subp(['qemu-img', 'create', '-f', 'qcow2', - '-b', src_img_path, img_path]) + subp.subp(['qemu-img', 'create', '-f', 'qcow2', + '-b', src_img_path, img_path]) return NoCloudKVMInstance(self, name, img_path, properties, config, features, user_data, meta_data) diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py index bebdf1c6..58f65e52 100644 --- a/tests/cloud_tests/platforms/platforms.py +++ b/tests/cloud_tests/platforms/platforms.py @@ -7,6 +7,7 @@ import shutil from simplestreams import filters, mirrors from simplestreams import util as s_util +from cloudinit import subp from cloudinit import util as c_util from tests.cloud_tests import util @@ -48,10 +49,10 @@ class Platform(object): if os.path.exists(filename): c_util.del_file(filename) - c_util.subp(['ssh-keygen', '-m', 'PEM', '-t', 'rsa', '-b', '4096', - '-f', filename, '-P', '', - '-C', 'ubuntu@cloud_test'], - capture=True) + subp.subp(['ssh-keygen', '-m', 'PEM', '-t', 'rsa', '-b', '4096', + '-f', filename, '-P', '', + '-C', 'ubuntu@cloud_test'], + capture=True) @staticmethod def _query_streams(img_conf, img_filter): diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py index e65771b1..7dcccbdd 100644 --- a/tests/cloud_tests/util.py +++ b/tests/cloud_tests/util.py @@ -17,6 +17,7 @@ import time import yaml from contextlib import contextmanager +from cloudinit import subp from cloudinit import util as c_util from tests.cloud_tests import LOG @@ -232,8 +233,8 @@ def flat_tar(output, basedir, owner='root', group='root'): @param group: group archive files belong to @return_value: none """ - c_util.subp(['tar', 'cf', output, '--owner', owner, '--group', group, - '-C', basedir] + rel_files(basedir), capture=True) + subp.subp(['tar', 'cf', output, '--owner', owner, '--group', group, + '-C', basedir] + rel_files(basedir), capture=True) def parse_conf_list(entries, valid=None, boolean=False): @@ -465,7 +466,7 @@ class TargetBase(object): return path -class InTargetExecuteError(c_util.ProcessExecutionError): +class InTargetExecuteError(subp.ProcessExecutionError): """Error type for in target commands that fail.""" default_desc = 'Unexpected error while running command.' diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 9045e743..c5675249 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -15,6 +15,7 @@ from cloudinit.tests.helpers import ( from cloudinit import handlers from cloudinit import helpers +from cloudinit import subp from cloudinit import util from cloudinit.handlers.cloud_config import CloudConfigPartHandler @@ -66,7 +67,7 @@ class TestUpstartJobPartHandler(FilesystemMockingTestCase): util.ensure_dir("/etc/upstart") with mock.patch(self.mpath + 'SUITABLE_UPSTART', return_value=True): - with mock.patch.object(util, 'subp') as m_subp: + with mock.patch.object(subp, 'subp') as m_subp: h = UpstartJobPartHandler(paths) h.handle_part('', handlers.CONTENT_START, None, None, None) diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index 3119bfac..fc59d1d5 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -15,6 +15,7 @@ import shutil import tempfile from cloudinit import helpers +from cloudinit import subp from cloudinit import util from cloudinit.tests.helpers import CiTestCase, mock @@ -286,7 +287,7 @@ class TestUserDataRhevm(CiTestCase): def test_modprobe_fails(self): '''Test user_data_rhevm() where modprobe fails.''' - self.m_modprobe_floppy.side_effect = util.ProcessExecutionError( + self.m_modprobe_floppy.side_effect = subp.ProcessExecutionError( "Failed modprobe") dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual(False, dsrc.user_data_rhevm()) @@ -294,7 +295,7 @@ class TestUserDataRhevm(CiTestCase): def test_no_modprobe_cmd(self): '''Test user_data_rhevm() with no modprobe command.''' - self.m_modprobe_floppy.side_effect = util.ProcessExecutionError( + self.m_modprobe_floppy.side_effect = subp.ProcessExecutionError( "No such file or dir") dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual(False, dsrc.user_data_rhevm()) @@ -302,7 +303,7 @@ class TestUserDataRhevm(CiTestCase): def test_udevadm_fails(self): '''Test user_data_rhevm() where udevadm fails.''' - self.m_udevadm_settle.side_effect = util.ProcessExecutionError( + self.m_udevadm_settle.side_effect = subp.ProcessExecutionError( "Failed settle.") dsrc = dsac.DataSourceAltCloud({}, None, self.paths) self.assertEqual(False, dsrc.user_data_rhevm()) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index b9f4d8fd..05552a1e 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -491,7 +491,7 @@ scbus-1 on xpt0 bus 0 (dsaz, 'get_hostname', mock.MagicMock()), (dsaz, 'set_hostname', mock.MagicMock()), (dsaz, 'get_metadata_from_fabric', self.get_metadata_from_fabric), - (dsaz.util, 'which', lambda x: True), + (dsaz.subp, 'which', lambda x: True), (dsaz.util, 'read_dmi_data', mock.MagicMock( side_effect=_dmi_mocks)), (dsaz.util, 'wait_for_files', mock.MagicMock( @@ -1267,20 +1267,20 @@ scbus-1 on xpt0 bus 0 expected_config['config'].append(blacklist_config) self.assertEqual(netconfig, expected_config) - @mock.patch(MOCKPATH + 'util.subp') - def test_get_hostname_with_no_args(self, subp): + @mock.patch(MOCKPATH + 'subp.subp') + def test_get_hostname_with_no_args(self, m_subp): dsaz.get_hostname() - subp.assert_called_once_with(("hostname",), capture=True) + m_subp.assert_called_once_with(("hostname",), capture=True) - @mock.patch(MOCKPATH + 'util.subp') - def test_get_hostname_with_string_arg(self, subp): + @mock.patch(MOCKPATH + 'subp.subp') + def test_get_hostname_with_string_arg(self, m_subp): dsaz.get_hostname(hostname_command="hostname") - subp.assert_called_once_with(("hostname",), capture=True) + m_subp.assert_called_once_with(("hostname",), capture=True) - @mock.patch(MOCKPATH + 'util.subp') - def test_get_hostname_with_iterable_arg(self, subp): + @mock.patch(MOCKPATH + 'subp.subp') + def test_get_hostname_with_iterable_arg(self, m_subp): dsaz.get_hostname(hostname_command=("hostname",)) - subp.assert_called_once_with(("hostname",), capture=True) + m_subp.assert_called_once_with(("hostname",), capture=True) class TestAzureBounce(CiTestCase): @@ -1302,7 +1302,7 @@ class TestAzureBounce(CiTestCase): mock.patch.object(dsaz, 'get_metadata_from_imds', mock.MagicMock(return_value={}))) self.patches.enter_context( - mock.patch.object(dsaz.util, 'which', lambda x: True)) + mock.patch.object(dsaz.subp, 'which', lambda x: True)) self.patches.enter_context(mock.patch.object( dsaz, '_get_random_seed', return_value='wild')) @@ -1331,7 +1331,7 @@ class TestAzureBounce(CiTestCase): self.set_hostname = self.patches.enter_context( mock.patch.object(dsaz, 'set_hostname')) self.subp = self.patches.enter_context( - mock.patch(MOCKPATH + 'util.subp')) + mock.patch(MOCKPATH + 'subp.subp')) self.find_fallback_nic = self.patches.enter_context( mock.patch('cloudinit.net.find_fallback_nic', return_value='eth9')) @@ -1414,7 +1414,7 @@ class TestAzureBounce(CiTestCase): cfg = {'hostname_bounce': {'policy': 'force'}} dsrc = self._get_ds(self.get_ovf_env_with_dscfg(host_name, cfg), agent_command=['not', '__builtin__']) - patch_path = MOCKPATH + 'util.which' + patch_path = MOCKPATH + 'subp.which' with mock.patch(patch_path) as m_which: m_which.return_value = None ret = self._get_and_setup(dsrc) @@ -1988,7 +1988,7 @@ class TestPreprovisioningPollIMDS(CiTestCase): self.assertEqual(report_ready_func.call_count, 0) -@mock.patch(MOCKPATH + 'util.subp') +@mock.patch(MOCKPATH + 'subp.subp') @mock.patch(MOCKPATH + 'util.write_file') @mock.patch(MOCKPATH + 'util.is_FreeBSD') @mock.patch('cloudinit.sources.helpers.netlink.' @@ -2159,7 +2159,7 @@ class TestWBIsPlatformViable(CiTestCase): {'os.path.exists': False, # Non-matching Azure chassis-asset-tag 'util.read_dmi_data': dsaz.AZURE_CHASSIS_ASSET_TAG + 'X', - 'util.which': None}, + 'subp.which': None}, dsaz._is_platform_viable, 'doesnotmatter')) self.assertIn( "DEBUG: Non-Azure DMI asset tag '{0}' discovered.\n".format( diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 6344b0bb..71ef57f0 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -287,7 +287,7 @@ class TestOpenSSLManager(CiTestCase): self.addCleanup(patches.close) self.subp = patches.enter_context( - mock.patch.object(azure_helper.util, 'subp')) + mock.patch.object(azure_helper.subp, 'subp')) try: self.open = patches.enter_context( mock.patch('__builtin__.open')) diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index 83c2f753..e68168f2 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -41,7 +41,7 @@ class TestCloudStackPasswordFetching(CiTestCase): def _set_password_server_response(self, response_string): subp = mock.MagicMock(return_value=(response_string, '')) self.patches.enter_context( - mock.patch('cloudinit.sources.DataSourceCloudStack.util.subp', + mock.patch('cloudinit.sources.DataSourceCloudStack.subp.subp', subp)) return subp diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 486a2345..3ef7a4b2 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -10,6 +10,7 @@ import os from collections import OrderedDict from textwrap import dedent +from cloudinit import subp from cloudinit import util from cloudinit.tests.helpers import CiTestCase, mock, wrap_and_call from cloudinit.helpers import Paths @@ -401,8 +402,8 @@ class TestTransportIso9660(CiTestCase): self.assertTrue(dsovf.maybe_cdrom_device('xvdza1')) -@mock.patch(MPATH + "util.which") -@mock.patch(MPATH + "util.subp") +@mock.patch(MPATH + "subp.which") +@mock.patch(MPATH + "subp.subp") class TestTransportVmwareGuestinfo(CiTestCase): """Test the com.vmware.guestInfo transport implemented in transport_vmware_guestinfo.""" @@ -420,7 +421,7 @@ class TestTransportVmwareGuestinfo(CiTestCase): def test_notfound_on_exit_code_1(self, m_subp, m_which): """If vmware-rpctool exits 1, then must return not found.""" m_which.return_value = self.rpctool_path - m_subp.side_effect = util.ProcessExecutionError( + m_subp.side_effect = subp.ProcessExecutionError( stdout="", stderr="No value found", exit_code=1, cmd=["unused"]) self.assertEqual(NOT_FOUND, dsovf.transport_vmware_guestinfo()) self.assertEqual(1, m_subp.call_count) @@ -442,7 +443,7 @@ class TestTransportVmwareGuestinfo(CiTestCase): def test_notfound_and_warns_on_unexpected_exit_code(self, m_subp, m_which): """If vmware-rpctool exits non zero or 1, warnings should be logged.""" m_which.return_value = self.rpctool_path - m_subp.side_effect = util.ProcessExecutionError( + m_subp.side_effect = subp.ProcessExecutionError( stdout=None, stderr="No value found", exit_code=2, cmd=["unused"]) self.assertEqual(NOT_FOUND, dsovf.transport_vmware_guestinfo()) self.assertEqual(1, m_subp.call_count) diff --git a/tests/unittests/test_datasource/test_rbx.py b/tests/unittests/test_datasource/test_rbx.py index 553af62e..d017510e 100644 --- a/tests/unittests/test_datasource/test_rbx.py +++ b/tests/unittests/test_datasource/test_rbx.py @@ -4,7 +4,7 @@ from cloudinit import helpers from cloudinit import distros from cloudinit.sources import DataSourceRbxCloud as ds from cloudinit.tests.helpers import mock, CiTestCase, populate_dir -from cloudinit import util +from cloudinit import subp DS_PATH = "cloudinit.sources.DataSourceRbxCloud" @@ -157,7 +157,7 @@ class TestRbxDataSource(CiTestCase): expected ) - @mock.patch(DS_PATH + '.util.subp') + @mock.patch(DS_PATH + '.subp.subp') def test_gratuitous_arp_run_standard_arping(self, m_subp): """Test handle run arping & parameters.""" items = [ @@ -183,7 +183,7 @@ class TestRbxDataSource(CiTestCase): ], m_subp.call_args_list ) - @mock.patch(DS_PATH + '.util.subp') + @mock.patch(DS_PATH + '.subp.subp') def test_handle_rhel_like_arping(self, m_subp): """Test handle on RHEL-like distros.""" items = [ @@ -201,8 +201,8 @@ class TestRbxDataSource(CiTestCase): ) @mock.patch( - DS_PATH + '.util.subp', - side_effect=util.ProcessExecutionError() + DS_PATH + '.subp.subp', + side_effect=subp.ProcessExecutionError() ) def test_continue_on_arping_error(self, m_subp): """Continue when command error""" diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index f1ab1d7a..5847a384 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -32,8 +32,8 @@ from cloudinit.sources.DataSourceSmartOS import ( from cloudinit.event import EventType from cloudinit import helpers as c_helpers -from cloudinit.util import ( - b64e, subp, ProcessExecutionError, which, write_file) +from cloudinit.util import (b64e, write_file) +from cloudinit.subp import (subp, ProcessExecutionError, which) from cloudinit.tests.helpers import ( CiTestCase, mock, FilesystemMockingTestCase, skipIf) @@ -667,7 +667,7 @@ class TestIdentifyFile(CiTestCase): with self.allow_subp(["file"]): self.assertEqual("text/plain", identify_file(fname)) - @mock.patch(DSMOS + ".util.subp") + @mock.patch(DSMOS + ".subp.subp") def test_returns_none_on_error(self, m_subp): """On 'file' execution error, None should be returned.""" m_subp.side_effect = ProcessExecutionError("FILE_FAILED", exit_code=99) diff --git a/tests/unittests/test_distros/test_create_users.py b/tests/unittests/test_distros/test_create_users.py index ef11784d..94ab052d 100644 --- a/tests/unittests/test_distros/test_create_users.py +++ b/tests/unittests/test_distros/test_create_users.py @@ -46,7 +46,7 @@ class MyBaseDistro(distros.Distro): @mock.patch("cloudinit.distros.util.system_is_snappy", return_value=False) -@mock.patch("cloudinit.distros.util.subp") +@mock.patch("cloudinit.distros.subp.subp") class TestCreateUser(CiTestCase): with_logs = True @@ -240,7 +240,7 @@ class TestCreateUser(CiTestCase): [mock.call(set(['auth1']), user), # not disabled mock.call(set(['key1']), 'foouser', options=disable_prefix)]) - @mock.patch("cloudinit.distros.util.which") + @mock.patch("cloudinit.distros.subp.which") def test_lock_with_usermod_if_no_passwd(self, m_which, m_subp, m_is_snappy): """Lock uses usermod --lock if no 'passwd' cmd available.""" @@ -250,7 +250,7 @@ class TestCreateUser(CiTestCase): [mock.call(['usermod', '--lock', 'bob'])], m_subp.call_args_list) - @mock.patch("cloudinit.distros.util.which") + @mock.patch("cloudinit.distros.subp.which") def test_lock_with_passwd_if_available(self, m_which, m_subp, m_is_snappy): """Lock with only passwd will use passwd.""" @@ -260,7 +260,7 @@ class TestCreateUser(CiTestCase): [mock.call(['passwd', '-l', 'bob'])], m_subp.call_args_list) - @mock.patch("cloudinit.distros.util.which") + @mock.patch("cloudinit.distros.subp.which") def test_lock_raises_runtime_if_no_commands(self, m_which, m_subp, m_is_snappy): """Lock with no commands available raises RuntimeError.""" diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py index da16a797..7ff8240b 100644 --- a/tests/unittests/test_distros/test_debian.py +++ b/tests/unittests/test_distros/test_debian.py @@ -5,7 +5,7 @@ from cloudinit import util from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock) -@mock.patch("cloudinit.distros.debian.util.subp") +@mock.patch("cloudinit.distros.debian.subp.subp") class TestDebianApplyLocale(FilesystemMockingTestCase): def setUp(self): diff --git a/tests/unittests/test_distros/test_freebsd.py b/tests/unittests/test_distros/test_freebsd.py index 8af253a2..be565b04 100644 --- a/tests/unittests/test_distros/test_freebsd.py +++ b/tests/unittests/test_distros/test_freebsd.py @@ -8,7 +8,7 @@ import os class TestDeviceLookUp(CiTestCase): - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_find_freebsd_part_label(self, mock_subp): glabel_out = ''' gptid/fa52d426-c337-11e6-8911-00155d4c5e47 N/A da0p1 @@ -19,7 +19,7 @@ gptid/fa52d426-c337-11e6-8911-00155d4c5e47 N/A da0p1 res = find_freebsd_part("/dev/label/rootfs") self.assertEqual("da0p2", res) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_find_freebsd_part_gpt(self, mock_subp): glabel_out = ''' gpt/bootfs N/A vtbd0p1 diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index 02b334e3..6b6c1566 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -245,7 +245,7 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): for d_name in ("ubuntu", "rhel"): cls = distros.fetch(d_name) d = cls(d_name, {}, None) - with mock.patch("cloudinit.util.subp") as m_subp: + with mock.patch("cloudinit.subp.subp") as m_subp: d.expire_passwd("myuser") m_subp.assert_called_once_with(["passwd", "--expire", "myuser"]) @@ -253,7 +253,7 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): """Test FreeBSD.expire_passwd uses the pw command.""" cls = distros.fetch("freebsd") d = cls("freebsd", {}, None) - with mock.patch("cloudinit.util.subp") as m_subp: + with mock.patch("cloudinit.subp.subp") as m_subp: d.expire_passwd("myuser") m_subp.assert_called_once_with( ["pw", "usermod", "myuser", "-p", "01-Jan-1970"]) diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 910207ca..8d7b09c8 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -12,6 +12,7 @@ from cloudinit import helpers from cloudinit import settings from cloudinit.tests.helpers import ( FilesystemMockingTestCase, dir2dict) +from cloudinit import subp from cloudinit import util @@ -688,6 +689,6 @@ class TestNetCfgDistroArch(TestNetCfgDistroBase): def get_mode(path, target=None): - return os.stat(util.target_path(target, path)).st_mode & 0o777 + return os.stat(subp.target_path(target, path)).st_mode & 0o777 # vi: ts=4 expandtab diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index a6faf0ef..fa48410a 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -307,7 +307,7 @@ class TestUGNormalize(TestCase): self.assertEqual({'default': False}, users['joe']) self.assertEqual({'default': False}, users['bob']) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_create_snap_user(self, mock_subp): mock_subp.side_effect = [('{"username": "joe", "ssh-key-count": 1}\n', '')] @@ -326,7 +326,7 @@ class TestUGNormalize(TestCase): mock_subp.assert_called_with(snapcmd, capture=True, logstring=snapcmd) self.assertEqual(username, 'joe') - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_create_snap_user_known(self, mock_subp): mock_subp.side_effect = [('{"username": "joe", "ssh-key-count": 1}\n', '')] @@ -348,7 +348,7 @@ class TestUGNormalize(TestCase): @mock.patch('cloudinit.util.system_is_snappy') @mock.patch('cloudinit.util.is_group') - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_add_user_on_snappy_system(self, mock_subp, mock_isgrp, mock_snappy): mock_isgrp.return_value = False diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 65a96090..fbb88e8a 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -6,6 +6,7 @@ import os from uuid import uuid4 from cloudinit import safeyaml +from cloudinit import subp from cloudinit import util from cloudinit.tests.helpers import ( CiTestCase, dir2dict, populate_dir, populate_dir_with_ts) @@ -160,8 +161,8 @@ class DsIdentifyBase(CiTestCase): rc = 0 try: - out, err = util.subp(['sh', '-c', '. %s' % wrap], capture=True) - except util.ProcessExecutionError as e: + out, err = subp.subp(['sh', '-c', '. %s' % wrap], capture=True) + except subp.ProcessExecutionError as e: rc = e.exit_code out = e.stdout err = e.stderr diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py index 69009a44..e5382544 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py @@ -13,6 +13,7 @@ from cloudinit import cloud from cloudinit import distros from cloudinit import helpers from cloudinit import templater +from cloudinit import subp from cloudinit import util from cloudinit.config import cc_apt_configure @@ -66,7 +67,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): """ def setUp(self): super(TestAptSourceConfigSourceList, self).setUp() - self.subp = util.subp + self.subp = subp.subp self.new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.new_root) @@ -176,7 +177,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # the second mock restores the original subp with mock.patch.object(util, 'write_file') as mockwrite: - with mock.patch.object(util, 'subp', self.subp): + with mock.patch.object(subp, 'subp', self.subp): with mock.patch.object(Distro, 'get_primary_arch', return_value='amd64'): cc_apt_configure.handle("notimportant", cfg, mycloud, diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py index 0aa3d51a..b96fd4d4 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py @@ -13,6 +13,7 @@ from unittest.mock import call from cloudinit import cloud from cloudinit import distros from cloudinit import helpers +from cloudinit import subp from cloudinit import util from cloudinit.config import cc_apt_configure @@ -94,7 +95,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): """TestAptSourceConfigSourceList - Class to test sources list rendering""" def setUp(self): super(TestAptSourceConfigSourceList, self).setUp() - self.subp = util.subp + self.subp = subp.subp self.new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.new_root) @@ -222,7 +223,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # the second mock restores the original subp with mock.patch.object(util, 'write_file') as mockwrite: - with mock.patch.object(util, 'subp', self.subp): + with mock.patch.object(subp, 'subp', self.subp): with mock.patch.object(Distro, 'get_primary_arch', return_value='amd64'): cc_apt_configure.handle("notimportant", cfg, mycloud, diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py index 866752ef..f2349157 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py @@ -14,6 +14,7 @@ from unittest.mock import call from cloudinit.config import cc_apt_configure from cloudinit import gpg +from cloudinit import subp from cloudinit import util from cloudinit.tests.helpers import TestCase @@ -271,7 +272,7 @@ class TestAptSourceConfig(TestCase): """ cfg = self.wrapv1conf(cfg) - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('fakekey 1234', '')) as mockobj: cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) @@ -356,7 +357,7 @@ class TestAptSourceConfig(TestCase): """ cfg = self.wrapv1conf([cfg]) - with mock.patch.object(util, 'subp') as mockobj: + with mock.patch.object(subp, 'subp') as mockobj: cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) mockobj.assert_called_with(['apt-key', 'add', '-'], @@ -398,7 +399,7 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile} cfg = self.wrapv1conf([cfg]) - with mock.patch.object(util, 'subp') as mockobj: + with mock.patch.object(subp, 'subp') as mockobj: cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) mockobj.assert_called_once_with(['apt-key', 'add', '-'], @@ -413,7 +414,7 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile} cfg = self.wrapv1conf([cfg]) - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('fakekey 1212', '')) as mockobj: cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) @@ -476,7 +477,7 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile} cfg = self.wrapv1conf([cfg]) - with mock.patch.object(util, 'subp') as mockobj: + with mock.patch.object(subp, 'subp') as mockobj: cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) mockobj.assert_called_once_with(['add-apt-repository', 'ppa:smoser/cloud-init-test'], @@ -495,7 +496,7 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile3} cfg = self.wrapv1conf([cfg1, cfg2, cfg3]) - with mock.patch.object(util, 'subp') as mockobj: + with mock.patch.object(subp, 'subp') as mockobj: cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test'], diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py index aefe26c4..220100e2 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -18,6 +18,7 @@ from cloudinit import cloud from cloudinit import distros from cloudinit import gpg from cloudinit import helpers +from cloudinit import subp from cloudinit import util from cloudinit.config import cc_apt_configure @@ -221,7 +222,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): """ params = self._get_default_params() - with mock.patch("cloudinit.util.subp", + with mock.patch("cloudinit.subp.subp", return_value=('fakekey 1234', '')) as mockobj: self._add_apt_sources(cfg, TARGET, template_params=params, aa_repo_match=self.matcher) @@ -296,7 +297,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): ' xenial main'), 'key': "fakekey 4321"}} - with mock.patch.object(util, 'subp') as mockobj: + with mock.patch.object(subp, 'subp') as mockobj: self._add_apt_sources(cfg, TARGET, template_params=params, aa_repo_match=self.matcher) @@ -318,7 +319,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): params = self._get_default_params() cfg = {self.aptlistfile: {'key': "fakekey 4242"}} - with mock.patch.object(util, 'subp') as mockobj: + with mock.patch.object(subp, 'subp') as mockobj: self._add_apt_sources(cfg, TARGET, template_params=params, aa_repo_match=self.matcher) @@ -333,7 +334,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): params = self._get_default_params() cfg = {self.aptlistfile: {'keyid': "03683F77"}} - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('fakekey 1212', '')) as mockobj: self._add_apt_sources(cfg, TARGET, template_params=params, aa_repo_match=self.matcher) @@ -416,7 +417,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): params = self._get_default_params() cfg = {self.aptlistfile: {'source': 'ppa:smoser/cloud-init-test'}} - with mock.patch("cloudinit.util.subp") as mockobj: + with mock.patch("cloudinit.subp.subp") as mockobj: self._add_apt_sources(cfg, TARGET, template_params=params, aa_repo_match=self.matcher) mockobj.assert_any_call(['add-apt-repository', @@ -432,7 +433,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): self.aptlistfile2: {'source': 'ppa:smoser/cloud-init-test2'}, self.aptlistfile3: {'source': 'ppa:smoser/cloud-init-test3'}} - with mock.patch("cloudinit.util.subp") as mockobj: + with mock.patch("cloudinit.subp.subp") as mockobj: self._add_apt_sources(cfg, TARGET, template_params=params, aa_repo_match=self.matcher) calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test'], @@ -996,7 +997,7 @@ deb http://ubuntu.com/ubuntu/ xenial-proposed main""") class TestDebconfSelections(TestCase): - @mock.patch("cloudinit.config.cc_apt_configure.util.subp") + @mock.patch("cloudinit.config.cc_apt_configure.subp.subp") def test_set_sel_appends_newline_if_absent(self, m_subp): """Automatically append a newline to debconf-set-selections config.""" selections = b'some/setting boolean true' @@ -1081,7 +1082,7 @@ class TestDebconfSelections(TestCase): self.assertTrue(m_get_inst.called) self.assertEqual(m_dpkg_r.call_count, 0) - @mock.patch("cloudinit.config.cc_apt_configure.util.subp") + @mock.patch("cloudinit.config.cc_apt_configure.subp.subp") def test_dpkg_reconfigure_does_reconfigure(self, m_subp): target = "/foo-target" @@ -1104,12 +1105,12 @@ class TestDebconfSelections(TestCase): 'cloud-init'] self.assertEqual(expected, found) - @mock.patch("cloudinit.config.cc_apt_configure.util.subp") + @mock.patch("cloudinit.config.cc_apt_configure.subp.subp") def test_dpkg_reconfigure_not_done_on_no_data(self, m_subp): cc_apt_configure.dpkg_reconfigure([]) m_subp.assert_not_called() - @mock.patch("cloudinit.config.cc_apt_configure.util.subp") + @mock.patch("cloudinit.config.cc_apt_configure.subp.subp") def test_dpkg_reconfigure_not_done_if_no_cleaners(self, m_subp): cc_apt_configure.dpkg_reconfigure(['pkgfoo', 'pkgbar']) m_subp.assert_not_called() diff --git a/tests/unittests/test_handler/test_handler_bootcmd.py b/tests/unittests/test_handler/test_handler_bootcmd.py index a76760fa..b53d60d4 100644 --- a/tests/unittests/test_handler/test_handler_bootcmd.py +++ b/tests/unittests/test_handler/test_handler_bootcmd.py @@ -2,7 +2,7 @@ from cloudinit.config.cc_bootcmd import handle, schema from cloudinit.sources import DataSourceNone -from cloudinit import (distros, helpers, cloud, util) +from cloudinit import (distros, helpers, cloud, subp, util) from cloudinit.tests.helpers import ( CiTestCase, mock, SchemaTestCaseMixin, skipUnlessJsonSchema) @@ -36,7 +36,7 @@ class TestBootcmd(CiTestCase): def setUp(self): super(TestBootcmd, self).setUp() - self.subp = util.subp + self.subp = subp.subp self.new_root = self.tmp_dir() def _get_cloud(self, distro): @@ -130,7 +130,7 @@ class TestBootcmd(CiTestCase): with mock.patch(self._etmpfile_path, FakeExtendedTempFile): with self.allow_subp(['/bin/sh']): - with self.assertRaises(util.ProcessExecutionError) as ctxt: + with self.assertRaises(subp.ProcessExecutionError) as ctxt: handle('does-not-matter', valid_config, cc, LOG, []) self.assertIn( 'Unexpected error while running command.\n' diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index 286ef771..db0cdf9b 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -3,6 +3,7 @@ from cloudinit import cloud from cloudinit.config import cc_ca_certs from cloudinit import helpers +from cloudinit import subp from cloudinit import util from cloudinit.tests.helpers import TestCase @@ -228,7 +229,7 @@ class TestAddCaCerts(TestCase): class TestUpdateCaCerts(unittest.TestCase): def test_commands(self): - with mock.patch.object(util, 'subp') as mockobj: + with mock.patch.object(subp, 'subp') as mockobj: cc_ca_certs.update_ca_certs() mockobj.assert_called_once_with( ["update-ca-certificates"], capture=False) @@ -250,7 +251,7 @@ class TestRemoveDefaultCaCerts(TestCase): mock.patch.object(util, 'delete_dir_contents')) mock_write = mocks.enter_context( mock.patch.object(util, 'write_file')) - mock_subp = mocks.enter_context(mock.patch.object(util, 'subp')) + mock_subp = mocks.enter_context(mock.patch.object(subp, 'subp')) cc_ca_certs.remove_default_ca_certs() diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 8c476418..7918c609 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -41,7 +41,7 @@ class TestInstallChefOmnibus(HttprettyTestCase): httpretty.GET, cc_chef.OMNIBUS_URL, body=response, status=200) ret = (None, None) # stdout, stderr but capture=False - with mock.patch("cloudinit.config.cc_chef.util.subp_blob_in_tempfile", + with mock.patch("cloudinit.config.cc_chef.subp_blob_in_tempfile", return_value=ret) as m_subp_blob: cc_chef.install_chef_from_omnibus() # admittedly whitebox, but assuming subp_blob_in_tempfile works @@ -52,7 +52,7 @@ class TestInstallChefOmnibus(HttprettyTestCase): m_subp_blob.call_args_list) @mock.patch('cloudinit.config.cc_chef.url_helper.readurl') - @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile') + @mock.patch('cloudinit.config.cc_chef.subp_blob_in_tempfile') def test_install_chef_from_omnibus_retries_url(self, m_subp_blob, m_rdurl): """install_chef_from_omnibus retries OMNIBUS_URL upon failure.""" @@ -81,7 +81,7 @@ class TestInstallChefOmnibus(HttprettyTestCase): m_subp_blob.call_args_list[0][1]) @mock.patch("cloudinit.config.cc_chef.OMNIBUS_URL", OMNIBUS_URL_HTTP) - @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile') + @mock.patch('cloudinit.config.cc_chef.subp_blob_in_tempfile') def test_install_chef_from_omnibus_has_omnibus_version(self, m_subp_blob): """install_chef_from_omnibus provides version arg to OMNIBUS_URL.""" chef_outfile = self.tmp_path('chef.out', self.new_root) diff --git a/tests/unittests/test_handler/test_handler_disk_setup.py b/tests/unittests/test_handler/test_handler_disk_setup.py index 0e51f17a..4f4a57fa 100644 --- a/tests/unittests/test_handler/test_handler_disk_setup.py +++ b/tests/unittests/test_handler/test_handler_disk_setup.py @@ -44,7 +44,7 @@ class TestGetMbrHddSize(TestCase): super(TestGetMbrHddSize, self).setUp() self.patches = ExitStack() self.subp = self.patches.enter_context( - mock.patch.object(cc_disk_setup.util, 'subp')) + mock.patch.object(cc_disk_setup.subp, 'subp')) def tearDown(self): super(TestGetMbrHddSize, self).tearDown() @@ -173,7 +173,7 @@ class TestUpdateFsSetupDevices(TestCase): @mock.patch('cloudinit.config.cc_disk_setup.find_device_node', return_value=('/dev/xdb1', False)) @mock.patch('cloudinit.config.cc_disk_setup.device_type', return_value=None) -@mock.patch('cloudinit.config.cc_disk_setup.util.subp', return_value=('', '')) +@mock.patch('cloudinit.config.cc_disk_setup.subp.subp', return_value=('', '')) class TestMkfsCommandHandling(CiTestCase): with_logs = True @@ -204,7 +204,7 @@ class TestMkfsCommandHandling(CiTestCase): subp.assert_called_once_with( 'mkfs -t ext4 -L with_cmd /dev/xdb1', shell=True) - @mock.patch('cloudinit.config.cc_disk_setup.util.which') + @mock.patch('cloudinit.config.cc_disk_setup.subp.which') def test_overwrite_and_extra_opts_without_cmd(self, m_which, subp, *args): """mkfs observes extra_opts and overwrite settings when cmd is not present.""" @@ -222,7 +222,7 @@ class TestMkfsCommandHandling(CiTestCase): '-L', 'without_cmd', '-F', 'are', 'added'], shell=False) - @mock.patch('cloudinit.config.cc_disk_setup.util.which') + @mock.patch('cloudinit.config.cc_disk_setup.subp.which') def test_mkswap(self, m_which, subp, *args): """mkfs observes extra_opts and overwrite settings when cmd is not present.""" diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index 501bcca5..7f039b79 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -2,7 +2,7 @@ from cloudinit import cloud from cloudinit.config import cc_growpart -from cloudinit import util +from cloudinit import subp from cloudinit.tests.helpers import TestCase @@ -95,7 +95,7 @@ class TestConfig(TestCase): @mock.patch.dict("os.environ", clear=True) def test_no_resizers_auto_is_fine(self): with mock.patch.object( - util, 'subp', + subp, 'subp', return_value=(HELP_GROWPART_NO_RESIZE, "")) as mockobj: config = {'growpart': {'mode': 'auto'}} @@ -109,7 +109,7 @@ class TestConfig(TestCase): @mock.patch.dict("os.environ", clear=True) def test_no_resizers_mode_growpart_is_exception(self): with mock.patch.object( - util, 'subp', + subp, 'subp', return_value=(HELP_GROWPART_NO_RESIZE, "")) as mockobj: config = {'growpart': {'mode': "growpart"}} self.assertRaises( @@ -122,7 +122,7 @@ class TestConfig(TestCase): @mock.patch.dict("os.environ", clear=True) def test_mode_auto_prefers_growpart(self): with mock.patch.object( - util, 'subp', + subp, 'subp', return_value=(HELP_GROWPART_RESIZE, "")) as mockobj: ret = cc_growpart.resizer_factory(mode="auto") self.assertIsInstance(ret, cc_growpart.ResizeGrowPart) @@ -133,7 +133,7 @@ class TestConfig(TestCase): @mock.patch.dict("os.environ", clear=True) def test_mode_auto_falls_back_to_gpart(self): with mock.patch.object( - util, 'subp', + subp, 'subp', return_value=("", HELP_GPART)) as mockobj: ret = cc_growpart.resizer_factory(mode="auto") self.assertIsInstance(ret, cc_growpart.ResizeGpart) diff --git a/tests/unittests/test_handler/test_handler_landscape.py b/tests/unittests/test_handler/test_handler_landscape.py index db92a7e2..7d165687 100644 --- a/tests/unittests/test_handler/test_handler_landscape.py +++ b/tests/unittests/test_handler/test_handler_landscape.py @@ -49,8 +49,8 @@ class TestLandscape(FilesystemMockingTestCase): "'landscape' key existed in config, but not a dict", str(context_manager.exception)) - @mock.patch('cloudinit.config.cc_landscape.util') - def test_handler_restarts_landscape_client(self, m_util): + @mock.patch('cloudinit.config.cc_landscape.subp') + def test_handler_restarts_landscape_client(self, m_subp): """handler restarts lansdscape-client after install.""" mycloud = self._get_cloud('ubuntu') cfg = {'landscape': {'client': {}}} @@ -60,7 +60,7 @@ class TestLandscape(FilesystemMockingTestCase): cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None) self.assertEqual( [mock.call(['service', 'landscape-client', 'restart'])], - m_util.subp.call_args_list) + m_subp.subp.call_args_list) def test_handler_installs_client_and_creates_config_file(self): """Write landscape client.conf and install landscape-client.""" diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index 407aa6c4..47e7d804 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -84,7 +84,7 @@ class TestLocale(t_help.FilesystemMockingTestCase): util.write_file(locale_conf, 'LANG="en_US.UTF-8"\n') cfg = {'locale': 'C.UTF-8'} cc = self._get_cloud('ubuntu') - with mock.patch('cloudinit.distros.debian.util.subp') as m_subp: + with mock.patch('cloudinit.distros.debian.subp.subp') as m_subp: with mock.patch('cloudinit.distros.debian.LOCALE_CONF_FN', locale_conf): cc_locale.handle('cc_locale', cfg, cc, LOG, []) diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 40b521e5..21011204 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -31,13 +31,13 @@ class TestLxd(t_help.CiTestCase): return cc @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") - @mock.patch("cloudinit.config.cc_lxd.util") - def test_lxd_init(self, mock_util, m_maybe_clean): + @mock.patch("cloudinit.config.cc_lxd.subp") + def test_lxd_init(self, mock_subp, m_maybe_clean): cc = self._get_cloud('ubuntu') - mock_util.which.return_value = True + mock_subp.which.return_value = True m_maybe_clean.return_value = None cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, []) - self.assertTrue(mock_util.which.called) + self.assertTrue(mock_subp.which.called) # no bridge config, so maybe_cleanup should not be called. self.assertFalse(m_maybe_clean.called) self.assertEqual( @@ -45,14 +45,14 @@ class TestLxd(t_help.CiTestCase): mock.call( ['lxd', 'init', '--auto', '--network-address=0.0.0.0', '--storage-backend=zfs', '--storage-pool=poolname'])], - mock_util.subp.call_args_list) + mock_subp.subp.call_args_list) @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") - @mock.patch("cloudinit.config.cc_lxd.util") - def test_lxd_install(self, mock_util, m_maybe_clean): + @mock.patch("cloudinit.config.cc_lxd.subp") + def test_lxd_install(self, mock_subp, m_maybe_clean): cc = self._get_cloud('ubuntu') cc.distro = mock.MagicMock() - mock_util.which.return_value = None + mock_subp.which.return_value = None cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, []) self.assertNotIn('WARN', self.logs.getvalue()) self.assertTrue(cc.distro.install_packages.called) @@ -62,23 +62,23 @@ class TestLxd(t_help.CiTestCase): self.assertEqual(sorted(install_pkg), ['lxd', 'zfsutils-linux']) @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") - @mock.patch("cloudinit.config.cc_lxd.util") - def test_no_init_does_nothing(self, mock_util, m_maybe_clean): + @mock.patch("cloudinit.config.cc_lxd.subp") + def test_no_init_does_nothing(self, mock_subp, m_maybe_clean): cc = self._get_cloud('ubuntu') cc.distro = mock.MagicMock() cc_lxd.handle('cc_lxd', {'lxd': {}}, cc, self.logger, []) self.assertFalse(cc.distro.install_packages.called) - self.assertFalse(mock_util.subp.called) + self.assertFalse(mock_subp.subp.called) self.assertFalse(m_maybe_clean.called) @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") - @mock.patch("cloudinit.config.cc_lxd.util") - def test_no_lxd_does_nothing(self, mock_util, m_maybe_clean): + @mock.patch("cloudinit.config.cc_lxd.subp") + def test_no_lxd_does_nothing(self, mock_subp, m_maybe_clean): cc = self._get_cloud('ubuntu') cc.distro = mock.MagicMock() cc_lxd.handle('cc_lxd', {'package_update': True}, cc, self.logger, []) self.assertFalse(cc.distro.install_packages.called) - self.assertFalse(mock_util.subp.called) + self.assertFalse(mock_subp.subp.called) self.assertFalse(m_maybe_clean.called) def test_lxd_debconf_new_full(self): diff --git a/tests/unittests/test_handler/test_handler_mcollective.py b/tests/unittests/test_handler/test_handler_mcollective.py index c013a538..6891e15f 100644 --- a/tests/unittests/test_handler/test_handler_mcollective.py +++ b/tests/unittests/test_handler/test_handler_mcollective.py @@ -136,8 +136,9 @@ class TestHandler(t_help.TestCase): cc = cloud.Cloud(ds, paths, {}, d, None) return cc + @t_help.mock.patch("cloudinit.config.cc_mcollective.subp") @t_help.mock.patch("cloudinit.config.cc_mcollective.util") - def test_mcollective_install(self, mock_util): + def test_mcollective_install(self, mock_util, mock_subp): cc = self._get_cloud('ubuntu') cc.distro = t_help.mock.MagicMock() mock_util.load_file.return_value = b"" @@ -147,8 +148,8 @@ class TestHandler(t_help.TestCase): install_pkg = cc.distro.install_packages.call_args_list[0][0][0] self.assertEqual(install_pkg, ('mcollective',)) - self.assertTrue(mock_util.subp.called) - self.assertEqual(mock_util.subp.call_args_list[0][0][0], + self.assertTrue(mock_subp.subp.called) + self.assertEqual(mock_subp.subp.call_args_list[0][0][0], ['service', 'mcollective', 'restart']) # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index 35e72bd1..80c53c83 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -155,8 +155,8 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase): 'mock_is_block_device', return_value=True) - self.add_patch('cloudinit.config.cc_mounts.util.subp', - 'm_util_subp') + self.add_patch('cloudinit.config.cc_mounts.subp.subp', + 'm_subp_subp') self.add_patch('cloudinit.config.cc_mounts.util.mounts', 'mock_util_mounts', @@ -268,7 +268,7 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase): fstab_new_content = fd.read() self.assertEqual(fstab_expected_content, fstab_new_content) cc_mounts.handle(None, cc, self.mock_cloud, self.mock_log, []) - self.m_util_subp.assert_has_calls([ + self.m_subp_subp.assert_has_calls([ mock.call(['mount', '-a']), mock.call(['systemctl', 'daemon-reload'])]) diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 463d892a..92a33ec1 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -83,50 +83,50 @@ class TestNtp(FilesystemMockingTestCase): ntpconfig['template_name'] = os.path.basename(confpath) return ntpconfig - @mock.patch("cloudinit.config.cc_ntp.util") - def test_ntp_install(self, mock_util): + @mock.patch("cloudinit.config.cc_ntp.subp") + def test_ntp_install(self, mock_subp): """ntp_install_client runs install_func when check_exe is absent.""" - mock_util.which.return_value = None # check_exe not found. + mock_subp.which.return_value = None # check_exe not found. install_func = mock.MagicMock() cc_ntp.install_ntp_client(install_func, packages=['ntpx'], check_exe='ntpdx') - mock_util.which.assert_called_with('ntpdx') + mock_subp.which.assert_called_with('ntpdx') install_func.assert_called_once_with(['ntpx']) - @mock.patch("cloudinit.config.cc_ntp.util") - def test_ntp_install_not_needed(self, mock_util): + @mock.patch("cloudinit.config.cc_ntp.subp") + def test_ntp_install_not_needed(self, mock_subp): """ntp_install_client doesn't install when check_exe is found.""" client = 'chrony' - mock_util.which.return_value = [client] # check_exe found. + mock_subp.which.return_value = [client] # check_exe found. install_func = mock.MagicMock() cc_ntp.install_ntp_client(install_func, packages=[client], check_exe=client) install_func.assert_not_called() - @mock.patch("cloudinit.config.cc_ntp.util") - def test_ntp_install_no_op_with_empty_pkg_list(self, mock_util): + @mock.patch("cloudinit.config.cc_ntp.subp") + def test_ntp_install_no_op_with_empty_pkg_list(self, mock_subp): """ntp_install_client runs install_func with empty list""" - mock_util.which.return_value = None # check_exe not found + mock_subp.which.return_value = None # check_exe not found install_func = mock.MagicMock() cc_ntp.install_ntp_client(install_func, packages=[], check_exe='timesyncd') install_func.assert_called_once_with([]) - @mock.patch("cloudinit.config.cc_ntp.util") - def test_reload_ntp_defaults(self, mock_util): + @mock.patch("cloudinit.config.cc_ntp.subp") + def test_reload_ntp_defaults(self, mock_subp): """Test service is restarted/reloaded (defaults)""" service = 'ntp_service_name' cmd = ['service', service, 'restart'] cc_ntp.reload_ntp(service) - mock_util.subp.assert_called_with(cmd, capture=True) + mock_subp.subp.assert_called_with(cmd, capture=True) - @mock.patch("cloudinit.config.cc_ntp.util") - def test_reload_ntp_systemd(self, mock_util): + @mock.patch("cloudinit.config.cc_ntp.subp") + def test_reload_ntp_systemd(self, mock_subp): """Test service is restarted/reloaded (systemd)""" service = 'ntp_service_name' cc_ntp.reload_ntp(service, systemd=True) cmd = ['systemctl', 'reload-or-restart', service] - mock_util.subp.assert_called_with(cmd, capture=True) + mock_subp.subp.assert_called_with(cmd, capture=True) def test_ntp_rename_ntp_conf(self): """When NTP_CONF exists, rename_ntp moves it.""" @@ -440,9 +440,10 @@ class TestNtp(FilesystemMockingTestCase): cc_ntp.handle('notimportant', cfg, mycloud, None, None) self.assertEqual(0, m_select.call_count) + @mock.patch("cloudinit.config.cc_ntp.subp") @mock.patch('cloudinit.config.cc_ntp.select_ntp_client') @mock.patch("cloudinit.distros.Distro.uses_systemd") - def test_ntp_the_whole_package(self, m_sysd, m_select): + def test_ntp_the_whole_package(self, m_sysd, m_select, m_subp): """Test enabled config renders template, and restarts service """ cfg = {'ntp': {'enabled': True}} for distro in cc_ntp.distros: @@ -458,12 +459,12 @@ class TestNtp(FilesystemMockingTestCase): # allow use of util.mergemanydict m_util.mergemanydict.side_effect = util.mergemanydict # default client is present - m_util.which.return_value = True + m_subp.which.return_value = True # use the config 'enabled' value m_util.is_false.return_value = util.is_false( cfg['ntp']['enabled']) cc_ntp.handle('notimportant', cfg, mycloud, None, None) - m_util.subp.assert_called_with( + m_subp.subp.assert_called_with( ['systemctl', 'reload-or-restart', service_name], capture=True) self.assertEqual( @@ -503,7 +504,7 @@ class TestNtp(FilesystemMockingTestCase): expected_client = mycloud.distro.preferred_ntp_clients[0] self.assertEqual('ntp', expected_client) - @mock.patch('cloudinit.config.cc_ntp.util.which') + @mock.patch('cloudinit.config.cc_ntp.subp.which') def test_snappy_system_picks_timesyncd(self, m_which): """Test snappy systems prefer installed clients""" @@ -528,7 +529,7 @@ class TestNtp(FilesystemMockingTestCase): self.assertEqual(sorted(expected_cfg), sorted(cfg)) self.assertEqual(sorted(expected_cfg), sorted(result)) - @mock.patch('cloudinit.config.cc_ntp.util.which') + @mock.patch('cloudinit.config.cc_ntp.subp.which') def test_ntp_distro_searches_all_preferred_clients(self, m_which): """Test select_ntp_client search all distro perferred clients """ # nothing is installed @@ -546,7 +547,7 @@ class TestNtp(FilesystemMockingTestCase): m_which.assert_has_calls(expected_calls) self.assertEqual(sorted(expected_cfg), sorted(cfg)) - @mock.patch('cloudinit.config.cc_ntp.util.which') + @mock.patch('cloudinit.config.cc_ntp.subp.which') def test_user_cfg_ntp_client_auto_uses_distro_clients(self, m_which): """Test user_cfg.ntp_client='auto' defaults to distro search""" # nothing is installed @@ -566,7 +567,7 @@ class TestNtp(FilesystemMockingTestCase): @mock.patch('cloudinit.config.cc_ntp.write_ntp_config_template') @mock.patch('cloudinit.cloud.Cloud.get_template_filename') - @mock.patch('cloudinit.config.cc_ntp.util.which') + @mock.patch('cloudinit.config.cc_ntp.subp.which') def test_ntp_custom_client_overrides_installed_clients(self, m_which, m_tmpfn, m_write): """Test user client is installed despite other clients present """ @@ -582,7 +583,7 @@ class TestNtp(FilesystemMockingTestCase): m_install.assert_called_with([client]) m_which.assert_called_with(client) - @mock.patch('cloudinit.config.cc_ntp.util.which') + @mock.patch('cloudinit.config.cc_ntp.subp.which') def test_ntp_system_config_overrides_distro_builtin_clients(self, m_which): """Test distro system_config overrides builtin preferred ntp clients""" system_client = 'chrony' @@ -597,7 +598,7 @@ class TestNtp(FilesystemMockingTestCase): self.assertEqual(sorted(expected_cfg), sorted(result)) m_which.assert_has_calls([]) - @mock.patch('cloudinit.config.cc_ntp.util.which') + @mock.patch('cloudinit.config.cc_ntp.subp.which') def test_ntp_user_config_overrides_system_cfg(self, m_which): """Test user-data overrides system_config ntp_client""" system_client = 'chrony' diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py index 2506d18a..62388ac6 100644 --- a/tests/unittests/test_handler/test_handler_puppet.py +++ b/tests/unittests/test_handler/test_handler_puppet.py @@ -12,11 +12,11 @@ import textwrap LOG = logging.getLogger(__name__) -@mock.patch('cloudinit.config.cc_puppet.util') +@mock.patch('cloudinit.config.cc_puppet.subp.subp') @mock.patch('cloudinit.config.cc_puppet.os') class TestAutostartPuppet(CiTestCase): - def test_wb_autostart_puppet_updates_puppet_default(self, m_os, m_util): + def test_wb_autostart_puppet_updates_puppet_default(self, m_os, m_subp): """Update /etc/default/puppet to autostart if it exists.""" def _fake_exists(path): @@ -27,9 +27,9 @@ class TestAutostartPuppet(CiTestCase): self.assertEqual( [mock.call(['sed', '-i', '-e', 's/^START=.*/START=yes/', '/etc/default/puppet'], capture=False)], - m_util.subp.call_args_list) + m_subp.call_args_list) - def test_wb_autostart_pupppet_enables_puppet_systemctl(self, m_os, m_util): + def test_wb_autostart_pupppet_enables_puppet_systemctl(self, m_os, m_subp): """If systemctl is present, enable puppet via systemctl.""" def _fake_exists(path): @@ -39,9 +39,9 @@ class TestAutostartPuppet(CiTestCase): cc_puppet._autostart_puppet(LOG) expected_calls = [mock.call( ['/bin/systemctl', 'enable', 'puppet.service'], capture=False)] - self.assertEqual(expected_calls, m_util.subp.call_args_list) + self.assertEqual(expected_calls, m_subp.call_args_list) - def test_wb_autostart_pupppet_enables_puppet_chkconfig(self, m_os, m_util): + def test_wb_autostart_pupppet_enables_puppet_chkconfig(self, m_os, m_subp): """If chkconfig is present, enable puppet via checkcfg.""" def _fake_exists(path): @@ -51,7 +51,7 @@ class TestAutostartPuppet(CiTestCase): cc_puppet._autostart_puppet(LOG) expected_calls = [mock.call( ['/sbin/chkconfig', 'puppet', 'on'], capture=False)] - self.assertEqual(expected_calls, m_util.subp.call_args_list) + self.assertEqual(expected_calls, m_subp.call_args_list) @mock.patch('cloudinit.config.cc_puppet._autostart_puppet') @@ -81,7 +81,7 @@ class TestPuppetHandle(CiTestCase): "no 'puppet' configuration found", self.logs.getvalue()) self.assertEqual(0, m_auto.call_count) - @mock.patch('cloudinit.config.cc_puppet.util.subp') + @mock.patch('cloudinit.config.cc_puppet.subp.subp') def test_handler_puppet_config_starts_puppet_service(self, m_subp, m_auto): """Cloud-config 'puppet' configuration starts puppet.""" mycloud = self._get_cloud('ubuntu') @@ -92,7 +92,7 @@ class TestPuppetHandle(CiTestCase): [mock.call(['service', 'puppet', 'start'], capture=False)], m_subp.call_args_list) - @mock.patch('cloudinit.config.cc_puppet.util.subp') + @mock.patch('cloudinit.config.cc_puppet.subp.subp') def test_handler_empty_puppet_config_installs_puppet(self, m_subp, m_auto): """Cloud-config empty 'puppet' configuration installs latest puppet.""" mycloud = self._get_cloud('ubuntu') @@ -103,7 +103,7 @@ class TestPuppetHandle(CiTestCase): [mock.call(('puppet', None))], mycloud.distro.install_packages.call_args_list) - @mock.patch('cloudinit.config.cc_puppet.util.subp') + @mock.patch('cloudinit.config.cc_puppet.subp.subp') def test_handler_puppet_config_installs_puppet_on_true(self, m_subp, _): """Cloud-config with 'puppet' key installs when 'install' is True.""" mycloud = self._get_cloud('ubuntu') @@ -114,7 +114,7 @@ class TestPuppetHandle(CiTestCase): [mock.call(('puppet', None))], mycloud.distro.install_packages.call_args_list) - @mock.patch('cloudinit.config.cc_puppet.util.subp') + @mock.patch('cloudinit.config.cc_puppet.subp.subp') def test_handler_puppet_config_installs_puppet_version(self, m_subp, _): """Cloud-config 'puppet' configuration can specify a version.""" mycloud = self._get_cloud('ubuntu') @@ -125,7 +125,7 @@ class TestPuppetHandle(CiTestCase): [mock.call(('puppet', '3.8'))], mycloud.distro.install_packages.call_args_list) - @mock.patch('cloudinit.config.cc_puppet.util.subp') + @mock.patch('cloudinit.config.cc_puppet.subp.subp') def test_handler_puppet_config_updates_puppet_conf(self, m_subp, m_auto): """When 'conf' is provided update values in PUPPET_CONF_PATH.""" mycloud = self._get_cloud('ubuntu') @@ -141,7 +141,7 @@ class TestPuppetHandle(CiTestCase): expected = '[agent]\nserver = puppetmaster.example.org\nother = 3\n\n' self.assertEqual(expected, content) - @mock.patch('cloudinit.config.cc_puppet.util.subp') + @mock.patch('cloudinit.config.cc_puppet.subp.subp') def test_handler_puppet_writes_csr_attributes_file(self, m_subp, m_auto): """When csr_attributes is provided creates file in PUPPET_CSR_ATTRIBUTES_PATH.""" diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py index 9ce334ac..73237d68 100644 --- a/tests/unittests/test_handler/test_handler_runcmd.py +++ b/tests/unittests/test_handler/test_handler_runcmd.py @@ -2,7 +2,7 @@ from cloudinit.config.cc_runcmd import handle, schema from cloudinit.sources import DataSourceNone -from cloudinit import (distros, helpers, cloud, util) +from cloudinit import (distros, helpers, cloud, subp, util) from cloudinit.tests.helpers import ( CiTestCase, FilesystemMockingTestCase, SchemaTestCaseMixin, skipUnlessJsonSchema) @@ -20,7 +20,7 @@ class TestRuncmd(FilesystemMockingTestCase): def setUp(self): super(TestRuncmd, self).setUp() - self.subp = util.subp + self.subp = subp.subp self.new_root = self.tmp_dir() def _get_cloud(self, distro): diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index abecc53b..85167f19 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -17,6 +17,7 @@ from io import BytesIO from cloudinit import cloud from cloudinit import distros from cloudinit import helpers +from cloudinit import subp from cloudinit import util from cloudinit.sources import DataSourceNone @@ -35,8 +36,8 @@ class TestRandomSeed(t_help.TestCase): self.unapply = [] # by default 'which' has nothing in its path - self.apply_patches([(util, 'which', self._which)]) - self.apply_patches([(util, 'subp', self._subp)]) + self.apply_patches([(subp, 'which', self._which)]) + self.apply_patches([(subp, 'subp', self._subp)]) self.subp_called = [] self.whichdata = {} diff --git a/tests/unittests/test_handler/test_handler_spacewalk.py b/tests/unittests/test_handler/test_handler_spacewalk.py index 410e6f77..26f7648f 100644 --- a/tests/unittests/test_handler/test_handler_spacewalk.py +++ b/tests/unittests/test_handler/test_handler_spacewalk.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit.config import cc_spacewalk -from cloudinit import util +from cloudinit import subp from cloudinit.tests import helpers @@ -19,20 +19,20 @@ class TestSpacewalk(helpers.TestCase): } } - @mock.patch("cloudinit.config.cc_spacewalk.util.subp") - def test_not_is_registered(self, mock_util_subp): - mock_util_subp.side_effect = util.ProcessExecutionError(exit_code=1) + @mock.patch("cloudinit.config.cc_spacewalk.subp.subp") + def test_not_is_registered(self, mock_subp): + mock_subp.side_effect = subp.ProcessExecutionError(exit_code=1) self.assertFalse(cc_spacewalk.is_registered()) - @mock.patch("cloudinit.config.cc_spacewalk.util.subp") - def test_is_registered(self, mock_util_subp): - mock_util_subp.side_effect = None + @mock.patch("cloudinit.config.cc_spacewalk.subp.subp") + def test_is_registered(self, mock_subp): + mock_subp.side_effect = None self.assertTrue(cc_spacewalk.is_registered()) - @mock.patch("cloudinit.config.cc_spacewalk.util.subp") - def test_do_register(self, mock_util_subp): + @mock.patch("cloudinit.config.cc_spacewalk.subp.subp") + def test_do_register(self, mock_subp): cc_spacewalk.do_register(**self.space_cfg['spacewalk']) - mock_util_subp.assert_called_with([ + mock_subp.assert_called_with([ 'rhnreg_ks', '--serverUrl', 'https://localhost/XMLRPC', '--profilename', 'test', diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 23626395..a56022ef 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -8,6 +8,7 @@ from cloudinit.net import ( renderers, sysconfig) from cloudinit.sources.helpers import openstack from cloudinit import temp_utils +from cloudinit import subp from cloudinit import util from cloudinit import safeyaml as yaml @@ -3192,7 +3193,7 @@ USERCTL=no def test_check_ifcfg_rh(self): """ifcfg-rh plugin is added NetworkManager.conf if conf present.""" render_dir = self.tmp_dir() - nm_cfg = util.target_path(render_dir, path=self.nm_cfg_file) + nm_cfg = subp.target_path(render_dir, path=self.nm_cfg_file) util.ensure_dir(os.path.dirname(nm_cfg)) # write a template nm.conf, note plugins is a list here @@ -3215,7 +3216,7 @@ USERCTL=no """ifcfg-rh plugin is append when plugins is a string.""" render_dir = self.tmp_path("render") os.makedirs(render_dir) - nm_cfg = util.target_path(render_dir, path=self.nm_cfg_file) + nm_cfg = subp.target_path(render_dir, path=self.nm_cfg_file) util.ensure_dir(os.path.dirname(nm_cfg)) # write a template nm.conf, note plugins is a value here @@ -3240,7 +3241,7 @@ USERCTL=no """enable_ifcfg_plugin creates plugins value if missing.""" render_dir = self.tmp_path("render") os.makedirs(render_dir) - nm_cfg = util.target_path(render_dir, path=self.nm_cfg_file) + nm_cfg = subp.target_path(render_dir, path=self.nm_cfg_file) util.ensure_dir(os.path.dirname(nm_cfg)) # write a template nm.conf, note plugins is missing @@ -3920,7 +3921,7 @@ class TestNetplanCleanDefault(CiTestCase): files = sorted(populate_dir(tmpd, content)) netplan._clean_default(target=tmpd) found = [t for t in files if os.path.exists(t)] - expected = [util.target_path(tmpd, f) for f in (astamp, anet, ayaml)] + expected = [subp.target_path(tmpd, f) for f in (astamp, anet, ayaml)] self.assertEqual(sorted(expected), found) @@ -3933,7 +3934,7 @@ class TestNetplanPostcommands(CiTestCase): @mock.patch.object(netplan.Renderer, '_netplan_generate') @mock.patch.object(netplan.Renderer, '_net_setup_link') - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_netplan_render_calls_postcmds(self, mock_subp, mock_netplan_generate, mock_net_setup_link): @@ -3947,7 +3948,7 @@ class TestNetplanPostcommands(CiTestCase): render_target = 'netplan.yaml' renderer = netplan.Renderer( {'netplan_path': render_target, 'postcmds': True}) - mock_subp.side_effect = iter([util.ProcessExecutionError]) + mock_subp.side_effect = iter([subp.ProcessExecutionError]) renderer.render_network_state(ns, target=render_dir) mock_netplan_generate.assert_called_with(run=True) @@ -3955,7 +3956,7 @@ class TestNetplanPostcommands(CiTestCase): @mock.patch('cloudinit.util.SeLinuxGuard') @mock.patch.object(netplan, "get_devicelist") - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_netplan_postcmds(self, mock_subp, mock_devlist, mock_sel): mock_sel.__enter__ = mock.Mock(return_value=False) mock_sel.__exit__ = mock.Mock() @@ -3971,7 +3972,7 @@ class TestNetplanPostcommands(CiTestCase): renderer = netplan.Renderer( {'netplan_path': render_target, 'postcmds': True}) mock_subp.side_effect = iter([ - util.ProcessExecutionError, + subp.ProcessExecutionError, ('', ''), ('', ''), ]) @@ -4260,7 +4261,7 @@ class TestNetplanRoundTrip(CiTestCase): def setUp(self): super(TestNetplanRoundTrip, self).setUp() - self.add_patch('cloudinit.net.netplan.util.subp', 'm_subp') + self.add_patch('cloudinit.net.netplan.subp.subp', 'm_subp') self.m_subp.return_value = (self.NETPLAN_INFO_OUT, '') def _render_and_read(self, network_config=None, state=None, @@ -5157,7 +5158,7 @@ def _gzip_data(data): class TestRenameInterfaces(CiTestCase): - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_rename_all(self, mock_subp): renames = [ ('00:11:22:33:44:55', 'interface0', 'virtio_net', '0x3'), @@ -5188,7 +5189,7 @@ class TestRenameInterfaces(CiTestCase): capture=True), ]) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_rename_no_driver_no_device_id(self, mock_subp): renames = [ ('00:11:22:33:44:55', 'interface0', None, None), @@ -5219,7 +5220,7 @@ class TestRenameInterfaces(CiTestCase): capture=True), ]) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_rename_all_bounce(self, mock_subp): renames = [ ('00:11:22:33:44:55', 'interface0', 'virtio_net', '0x3'), @@ -5254,7 +5255,7 @@ class TestRenameInterfaces(CiTestCase): mock.call(['ip', 'link', 'set', 'interface2', 'up'], capture=True) ]) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_rename_duplicate_macs(self, mock_subp): renames = [ ('00:11:22:33:44:55', 'eth0', 'hv_netsvc', '0x3'), @@ -5283,7 +5284,7 @@ class TestRenameInterfaces(CiTestCase): capture=True), ]) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_rename_duplicate_macs_driver_no_devid(self, mock_subp): renames = [ ('00:11:22:33:44:55', 'eth0', 'hv_netsvc', None), @@ -5312,7 +5313,7 @@ class TestRenameInterfaces(CiTestCase): capture=True), ]) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_rename_multi_mac_dups(self, mock_subp): renames = [ ('00:11:22:33:44:55', 'eth0', 'hv_netsvc', '0x3'), @@ -5351,7 +5352,7 @@ class TestRenameInterfaces(CiTestCase): capture=True), ]) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_rename_macs_case_insensitive(self, mock_subp): """_rename_interfaces must support upper or lower case macs.""" renames = [ diff --git a/tests/unittests/test_net_freebsd.py b/tests/unittests/test_net_freebsd.py index 48296c30..414b4830 100644 --- a/tests/unittests/test_net_freebsd.py +++ b/tests/unittests/test_net_freebsd.py @@ -7,7 +7,7 @@ SAMPLE_FREEBSD_IFCONFIG_OUT = readResource("netinfo/freebsd-ifconfig-output") class TestInterfacesByMac(CiTestCase): - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') @mock.patch('cloudinit.util.is_FreeBSD') def test_get_interfaces_by_mac(self, mock_is_FreeBSD, mock_subp): mock_is_FreeBSD.return_value = True diff --git a/tests/unittests/test_render_cloudcfg.py b/tests/unittests/test_render_cloudcfg.py index 8b1e6042..393a78b1 100644 --- a/tests/unittests/test_render_cloudcfg.py +++ b/tests/unittests/test_render_cloudcfg.py @@ -5,6 +5,7 @@ import sys import pytest +from cloudinit import subp from cloudinit import util # TODO(Look to align with tools.render-cloudcfg or cloudinit.distos.OSFAMILIES) @@ -20,7 +21,7 @@ class TestRenderCloudCfg: @pytest.mark.parametrize('variant', (DISTRO_VARIANTS)) def test_variant_sets_distro_in_cloud_cfg(self, variant, tmpdir): outfile = tmpdir.join('outcfg').strpath - util.subp( + subp.subp( self.cmd + ['--variant', variant, self.tmpl_path, outfile]) with open(outfile) as stream: system_cfg = util.load_yaml(stream.read()) @@ -31,7 +32,7 @@ class TestRenderCloudCfg: @pytest.mark.parametrize('variant', (DISTRO_VARIANTS)) def test_variant_sets_default_user_in_cloud_cfg(self, variant, tmpdir): outfile = tmpdir.join('outcfg').strpath - util.subp( + subp.subp( self.cmd + ['--variant', variant, self.tmpl_path, outfile]) with open(outfile) as stream: system_cfg = util.load_yaml(stream.read()) @@ -49,7 +50,7 @@ class TestRenderCloudCfg: self, variant, renderers, tmpdir ): outfile = tmpdir.join('outcfg').strpath - util.subp( + subp.subp( self.cmd + ['--variant', variant, self.tmpl_path, outfile]) with open(outfile) as stream: system_cfg = util.load_yaml(stream.read()) diff --git a/tests/unittests/test_reporting_hyperv.py b/tests/unittests/test_reporting_hyperv.py index b3e083c6..fa8f8859 100644 --- a/tests/unittests/test_reporting_hyperv.py +++ b/tests/unittests/test_reporting_hyperv.py @@ -131,7 +131,7 @@ class TextKvpReporter(CiTestCase): self.assertEqual(0, len(kvps)) @mock.patch('cloudinit.distros.uses_systemd') - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_get_boot_telemetry(self, m_subp, m_sysd): reporter = HyperVKvpReportingHandler(kvp_file_path=self.tmp_file_path) datetime_pattern = r"\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]" diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index 4cd27eed..53d3cd5a 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -6,7 +6,7 @@ import copy import logging from cloudinit.config import cc_rh_subscription -from cloudinit import util +from cloudinit import subp from cloudinit.tests.helpers import CiTestCase, mock @@ -56,7 +56,7 @@ class GoodTests(CiTestCase): ''' reg = "The system has been registered with ID:" \ " 12345678-abde-abcde-1234-1234567890abc" - m_sman_cli.side_effect = [util.ProcessExecutionError, (reg, 'bar')] + m_sman_cli.side_effect = [subp.ProcessExecutionError, (reg, 'bar')] self.handle(self.name, self.config, self.cloud_init, self.log, self.args) self.assertIn(mock.call(['identity']), m_sman_cli.call_args_list) @@ -93,7 +93,7 @@ class GoodTests(CiTestCase): reg = "The system has been registered with ID:" \ " 12345678-abde-abcde-1234-1234567890abc" m_sman_cli.side_effect = [ - util.ProcessExecutionError, + subp.ProcessExecutionError, (reg, 'bar'), ('Service level set to: self-support', ''), ('pool1\npool3\n', ''), ('pool2\n', ''), ('', ''), @@ -161,7 +161,7 @@ class TestBadInput(CiTestCase): def test_no_password(self, m_sman_cli): '''Attempt to register without the password key/value.''' - m_sman_cli.side_effect = [util.ProcessExecutionError, + m_sman_cli.side_effect = [subp.ProcessExecutionError, (self.reg, 'bar')] self.handle(self.name, self.config_no_password, self.cloud_init, self.log, self.args) @@ -169,7 +169,7 @@ class TestBadInput(CiTestCase): def test_no_org(self, m_sman_cli): '''Attempt to register without the org key/value.''' - m_sman_cli.side_effect = [util.ProcessExecutionError] + m_sman_cli.side_effect = [subp.ProcessExecutionError] self.handle(self.name, self.config_no_key, self.cloud_init, self.log, self.args) m_sman_cli.assert_called_with(['identity']) @@ -182,7 +182,7 @@ class TestBadInput(CiTestCase): def test_service_level_without_auto(self, m_sman_cli): '''Attempt to register using service-level without auto-attach key.''' - m_sman_cli.side_effect = [util.ProcessExecutionError, + m_sman_cli.side_effect = [subp.ProcessExecutionError, (self.reg, 'bar')] self.handle(self.name, self.config_service, self.cloud_init, self.log, self.args) @@ -195,7 +195,7 @@ class TestBadInput(CiTestCase): ''' Register with pools that are not in the format of a list ''' - m_sman_cli.side_effect = [util.ProcessExecutionError, + m_sman_cli.side_effect = [subp.ProcessExecutionError, (self.reg, 'bar')] self.handle(self.name, self.config_badpool, self.cloud_init, self.log, self.args) @@ -208,7 +208,7 @@ class TestBadInput(CiTestCase): ''' Register with repos that are not in the format of a list ''' - m_sman_cli.side_effect = [util.ProcessExecutionError, + m_sman_cli.side_effect = [subp.ProcessExecutionError, (self.reg, 'bar')] self.handle(self.name, self.config_badrepo, self.cloud_init, self.log, self.args) @@ -222,7 +222,7 @@ class TestBadInput(CiTestCase): ''' Attempt to register with a key that we don't know ''' - m_sman_cli.side_effect = [util.ProcessExecutionError, + m_sman_cli.side_effect = [subp.ProcessExecutionError, (self.reg, 'bar')] self.handle(self.name, self.config_badkey, self.cloud_init, self.log, self.args) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 4b439267..737dda8b 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -1,25 +1,20 @@ # This file is part of cloud-init. See LICENSE file for license information. import io -import json import logging import os import re import shutil import stat -import sys import tempfile import yaml from unittest import mock +from cloudinit import subp from cloudinit import importer, util from cloudinit.tests import helpers -BASH = util.which('bash') -BOGUS_COMMAND = 'this-is-not-expected-to-be-a-program-name' - - class FakeSelinux(object): def __init__(self, match_what): @@ -385,7 +380,7 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): self.assertEqual(expected, util.parse_mount_info('/run/lock', lines)) @mock.patch('cloudinit.util.os') - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_get_device_info_from_zpool(self, zpool_output, m_os): # mock /dev/zfs exists m_os.path.exists.return_value = True @@ -408,17 +403,17 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): self.assertIsNone(ret) @mock.patch('cloudinit.util.os') - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_get_device_info_from_zpool_handles_no_zpool(self, m_sub, m_os): """Handle case where there is no zpool command""" # mock /dev/zfs exists m_os.path.exists.return_value = True - m_sub.side_effect = util.ProcessExecutionError("No zpool cmd") + m_sub.side_effect = subp.ProcessExecutionError("No zpool cmd") ret = util.get_device_info_from_zpool('vmzroot') self.assertIsNone(ret) @mock.patch('cloudinit.util.os') - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_get_device_info_from_zpool_on_error(self, zpool_output, m_os): # mock /dev/zfs exists m_os.path.exists.return_value = True @@ -430,7 +425,7 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): ret = util.get_device_info_from_zpool('vmzroot') self.assertIsNone(ret) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_parse_mount_with_ext(self, mount_out): mount_out.return_value = ( helpers.readResource('mount_parse_ext.txt'), '') @@ -447,7 +442,7 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): ret = util.parse_mount('/not/existing/mount') self.assertIsNone(ret) - @mock.patch('cloudinit.util.subp') + @mock.patch('cloudinit.subp.subp') def test_parse_mount_with_zfs(self, mount_out): mount_out.return_value = ( helpers.readResource('mount_parse_zfs.txt'), '') @@ -513,13 +508,13 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): """ def _dmidecode_subp(cmd): if cmd[-1] != key: - raise util.ProcessExecutionError() + raise subp.ProcessExecutionError() return (content, error) self.patched_funcs.enter_context( - mock.patch.object(util, 'which', lambda _: True)) + mock.patch("cloudinit.subp.which", side_effect=lambda _: True)) self.patched_funcs.enter_context( - mock.patch.object(util, 'subp', _dmidecode_subp)) + mock.patch("cloudinit.subp.subp", side_effect=_dmidecode_subp)) def patch_mapping(self, new_mapping): self.patched_funcs.enter_context( @@ -546,10 +541,12 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): def test_dmidecode_not_used_on_arm(self): self.patch_mapping({}) + print("current =%s", subp) self._create_sysfs_parent_directory() dmi_val = 'from-dmidecode' dmi_name = 'use-dmidecode' self._configure_dmidecode_return(dmi_name, dmi_val) + print("now =%s", subp) expected = {'armel': None, 'aarch64': dmi_val, 'x86_64': dmi_val} found = {} @@ -560,6 +557,7 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): for arch in expected: m_uname.return_value = ('x-sysname', 'x-nodename', 'x-release', 'x-version', arch) + print("now2 =%s", subp) found[arch] = util.read_dmi_data(dmi_name) self.assertEqual(expected, found) @@ -570,7 +568,7 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): def test_none_returned_if_dmidecode_not_in_path(self): self.patched_funcs.enter_context( - mock.patch.object(util, 'which', lambda _: False)) + mock.patch.object(subp, 'which', lambda _: False)) self.patch_mapping({}) self.assertIsNone(util.read_dmi_data('expect-fail')) @@ -734,218 +732,6 @@ class TestReadSeeded(helpers.TestCase): self.assertEqual(found_ud, ud) -class TestSubp(helpers.CiTestCase): - allowed_subp = [BASH, 'cat', helpers.CiTestCase.SUBP_SHELL_TRUE, - BOGUS_COMMAND, sys.executable] - - stdin2err = [BASH, '-c', 'cat >&2'] - stdin2out = ['cat'] - utf8_invalid = b'ab\xaadef' - utf8_valid = b'start \xc3\xa9 end' - utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7' - printenv = [BASH, '-c', 'for n in "$@"; do echo "$n=${!n}"; done', '--'] - - def printf_cmd(self, *args): - # bash's printf supports \xaa. So does /usr/bin/printf - # but by using bash, we remove dependency on another program. - return([BASH, '-c', 'printf "$@"', 'printf'] + list(args)) - - def test_subp_handles_bytestrings(self): - """subp can run a bytestring command if shell is True.""" - tmp_file = self.tmp_path('test.out') - cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file) - (out, _err) = util.subp(cmd.encode('utf-8'), shell=True) - self.assertEqual(u'', out) - self.assertEqual(u'', _err) - self.assertEqual('HI MOM\n', util.load_file(tmp_file)) - - def test_subp_handles_strings(self): - """subp can run a string command if shell is True.""" - tmp_file = self.tmp_path('test.out') - cmd = 'echo HI MOM >> {tmp_file}'.format(tmp_file=tmp_file) - (out, _err) = util.subp(cmd, shell=True) - self.assertEqual(u'', out) - self.assertEqual(u'', _err) - self.assertEqual('HI MOM\n', util.load_file(tmp_file)) - - def test_subp_handles_utf8(self): - # The given bytes contain utf-8 accented characters as seen in e.g. - # the "deja dup" package in Ubuntu. - cmd = self.printf_cmd(self.utf8_valid_2) - (out, _err) = util.subp(cmd, capture=True) - self.assertEqual(out, self.utf8_valid_2.decode('utf-8')) - - def test_subp_respects_decode_false(self): - (out, err) = util.subp(self.stdin2out, capture=True, decode=False, - data=self.utf8_valid) - self.assertTrue(isinstance(out, bytes)) - self.assertTrue(isinstance(err, bytes)) - self.assertEqual(out, self.utf8_valid) - - def test_subp_decode_ignore(self): - # this executes a string that writes invalid utf-8 to stdout - (out, _err) = util.subp(self.printf_cmd('abc\\xaadef'), - capture=True, decode='ignore') - self.assertEqual(out, 'abcdef') - - def test_subp_decode_strict_valid_utf8(self): - (out, _err) = util.subp(self.stdin2out, capture=True, - decode='strict', data=self.utf8_valid) - self.assertEqual(out, self.utf8_valid.decode('utf-8')) - - def test_subp_decode_invalid_utf8_replaces(self): - (out, _err) = util.subp(self.stdin2out, capture=True, - data=self.utf8_invalid) - expected = self.utf8_invalid.decode('utf-8', 'replace') - self.assertEqual(out, expected) - - def test_subp_decode_strict_raises(self): - args = [] - kwargs = {'args': self.stdin2out, 'capture': True, - 'decode': 'strict', 'data': self.utf8_invalid} - self.assertRaises(UnicodeDecodeError, util.subp, *args, **kwargs) - - def test_subp_capture_stderr(self): - data = b'hello world' - (out, err) = util.subp(self.stdin2err, capture=True, - decode=False, data=data, - update_env={'LC_ALL': 'C'}) - self.assertEqual(err, data) - self.assertEqual(out, b'') - - def test_subp_reads_env(self): - with mock.patch.dict("os.environ", values={'FOO': 'BAR'}): - out, _err = util.subp(self.printenv + ['FOO'], capture=True) - self.assertEqual('FOO=BAR', out.splitlines()[0]) - - def test_subp_env_and_update_env(self): - out, _err = util.subp( - self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True, - env={'FOO': 'BAR'}, - update_env={'HOME': '/myhome', 'K2': 'V2'}) - self.assertEqual( - ['FOO=BAR', 'HOME=/myhome', 'K1=', 'K2=V2'], out.splitlines()) - - def test_subp_update_env(self): - extra = {'FOO': 'BAR', 'HOME': '/root', 'K1': 'V1'} - with mock.patch.dict("os.environ", values=extra): - out, _err = util.subp( - self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True, - update_env={'HOME': '/myhome', 'K2': 'V2'}) - - self.assertEqual( - ['FOO=BAR', 'HOME=/myhome', 'K1=V1', 'K2=V2'], out.splitlines()) - - def test_subp_warn_missing_shebang(self): - """Warn on no #! in script""" - noshebang = self.tmp_path('noshebang') - util.write_file(noshebang, 'true\n') - - os.chmod(noshebang, os.stat(noshebang).st_mode | stat.S_IEXEC) - with self.allow_subp([noshebang]): - self.assertRaisesRegex(util.ProcessExecutionError, - r'Missing #! in script\?', - util.subp, (noshebang,)) - - def test_subp_combined_stderr_stdout(self): - """Providing combine_capture as True redirects stderr to stdout.""" - data = b'hello world' - (out, err) = util.subp(self.stdin2err, capture=True, - combine_capture=True, decode=False, data=data) - self.assertEqual(b'', err) - self.assertEqual(data, out) - - def test_returns_none_if_no_capture(self): - (out, err) = util.subp(self.stdin2out, data=b'', capture=False) - self.assertIsNone(err) - self.assertIsNone(out) - - def test_exception_has_out_err_are_bytes_if_decode_false(self): - """Raised exc should have stderr, stdout as bytes if no decode.""" - with self.assertRaises(util.ProcessExecutionError) as cm: - util.subp([BOGUS_COMMAND], decode=False) - self.assertTrue(isinstance(cm.exception.stdout, bytes)) - self.assertTrue(isinstance(cm.exception.stderr, bytes)) - - def test_exception_has_out_err_are_bytes_if_decode_true(self): - """Raised exc should have stderr, stdout as string if no decode.""" - with self.assertRaises(util.ProcessExecutionError) as cm: - util.subp([BOGUS_COMMAND], decode=True) - self.assertTrue(isinstance(cm.exception.stdout, str)) - self.assertTrue(isinstance(cm.exception.stderr, str)) - - def test_bunch_of_slashes_in_path(self): - self.assertEqual("/target/my/path/", - util.target_path("/target/", "//my/path/")) - self.assertEqual("/target/my/path/", - util.target_path("/target/", "///my/path/")) - - def test_c_lang_can_take_utf8_args(self): - """Independent of system LC_CTYPE, args can contain utf-8 strings. - - When python starts up, its default encoding gets set based on - the value of LC_CTYPE. If no system locale is set, the default - encoding for both python2 and python3 in some paths will end up - being ascii. - - Attempts to use setlocale or patching (or changing) os.environ - in the current environment seem to not be effective. - - This test starts up a python with LC_CTYPE set to C so that - the default encoding will be set to ascii. In such an environment - Popen(['command', 'non-ascii-arg']) would cause a UnicodeDecodeError. - """ - python_prog = '\n'.join([ - 'import json, sys', - 'from cloudinit.util import subp', - 'data = sys.stdin.read()', - 'cmd = json.loads(data)', - 'subp(cmd, capture=False)', - '']) - cmd = [BASH, '-c', 'echo -n "$@"', '--', - self.utf8_valid.decode("utf-8")] - python_subp = [sys.executable, '-c', python_prog] - - out, _err = util.subp( - python_subp, update_env={'LC_CTYPE': 'C'}, - data=json.dumps(cmd).encode("utf-8"), - decode=False) - self.assertEqual(self.utf8_valid, out) - - def test_bogus_command_logs_status_messages(self): - """status_cb gets status messages logs on bogus commands provided.""" - logs = [] - - def status_cb(log): - logs.append(log) - - with self.assertRaises(util.ProcessExecutionError): - util.subp([BOGUS_COMMAND], status_cb=status_cb) - - expected = [ - 'Begin run command: {cmd}\n'.format(cmd=BOGUS_COMMAND), - 'ERROR: End run command: invalid command provided\n'] - self.assertEqual(expected, logs) - - def test_command_logs_exit_codes_to_status_cb(self): - """status_cb gets status messages containing command exit code.""" - logs = [] - - def status_cb(log): - logs.append(log) - - with self.assertRaises(util.ProcessExecutionError): - util.subp([BASH, '-c', 'exit 2'], status_cb=status_cb) - util.subp([BASH, '-c', 'exit 0'], status_cb=status_cb) - - expected = [ - 'Begin run command: %s -c exit 2\n' % BASH, - 'ERROR: End run command: exit(2)\n', - 'Begin run command: %s -c exit 0\n' % BASH, - 'End run command: exit(0)\n'] - self.assertEqual(expected, logs) - - class TestEncode(helpers.TestCase): """Test the encoding functions""" def test_decode_binary_plain_text_with_hex(self): @@ -966,7 +752,7 @@ class TestProcessExecutionError(helpers.TestCase): empty_description = 'Unexpected error while running command.' def test_pexec_error_indent_text(self): - error = util.ProcessExecutionError() + error = subp.ProcessExecutionError() msg = 'abc\ndef' formatted = 'abc\n{0}def'.format(' ' * 4) self.assertEqual(error._indent_text(msg, indent_level=4), formatted) @@ -976,10 +762,10 @@ class TestProcessExecutionError(helpers.TestCase): error._indent_text(msg.encode()), type(msg.encode())) def test_pexec_error_type(self): - self.assertIsInstance(util.ProcessExecutionError(), IOError) + self.assertIsInstance(subp.ProcessExecutionError(), IOError) def test_pexec_error_empty_msgs(self): - error = util.ProcessExecutionError() + error = subp.ProcessExecutionError() self.assertTrue(all(attr == self.empty_attr for attr in (error.stderr, error.stdout, error.reason))) self.assertEqual(error.description, self.empty_description) @@ -993,7 +779,7 @@ class TestProcessExecutionError(helpers.TestCase): stderr_msg = 'error error' cmd = 'test command' exit_code = 3 - error = util.ProcessExecutionError( + error = subp.ProcessExecutionError( stdout=stdout_msg, stderr=stderr_msg, exit_code=3, cmd=cmd) self.assertEqual(str(error), self.template.format( description=self.empty_description, stdout=stdout_msg, @@ -1004,7 +790,7 @@ class TestProcessExecutionError(helpers.TestCase): # make sure bytes is converted handled properly when formatting stdout_msg = 'multi\nline\noutput message'.encode() stderr_msg = 'multi\nline\nerror message\n\n\n' - error = util.ProcessExecutionError( + error = subp.ProcessExecutionError( stdout=stdout_msg, stderr=stderr_msg) self.assertEqual( str(error), @@ -1170,7 +956,7 @@ class TestGetProcEnv(helpers.TestCase): self.assertEqual(my_ppid, util.get_proc_ppid(my_pid)) -@mock.patch('cloudinit.util.subp') +@mock.patch('cloudinit.subp.subp') def test_find_devs_with_openbsd(m_subp): m_subp.return_value = ( 'cd0:,sd0:630d98d32b5d3759,sd1:,fd0:', '' @@ -1179,7 +965,7 @@ def test_find_devs_with_openbsd(m_subp): assert devlist == ['/dev/cd0a', '/dev/sd1i'] -@mock.patch('cloudinit.util.subp') +@mock.patch('cloudinit.subp.subp') def test_find_devs_with_openbsd_with_criteria(m_subp): m_subp.return_value = ( 'cd0:,sd0:630d98d32b5d3759,sd1:,fd0:', '' @@ -1209,7 +995,7 @@ def test_find_devs_with_freebsd(m_glob): assert devlist == ['/dev/msdosfs/EFISYS'] -@mock.patch("cloudinit.util.subp") +@mock.patch("cloudinit.subp.subp") def test_find_devs_with_netbsd(m_subp): side_effect_values = [ ("ld0 dk0 dk1 cd0", ""), diff --git a/tests/unittests/test_vmware/test_guestcust_util.py b/tests/unittests/test_vmware/test_guestcust_util.py index 394bee9f..c8b59d83 100644 --- a/tests/unittests/test_vmware/test_guestcust_util.py +++ b/tests/unittests/test_vmware/test_guestcust_util.py @@ -5,7 +5,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import util +from cloudinit import subp from cloudinit.sources.helpers.vmware.imc.config import Config from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile from cloudinit.sources.helpers.vmware.imc.guestcust_util import ( @@ -21,7 +21,7 @@ class TestGuestCustUtil(CiTestCase): This test is designed to verify the behavior if vmware-toolbox-cmd is not installed. """ - with mock.patch.object(util, 'which', return_value=None): + with mock.patch.object(subp, 'which', return_value=None): self.assertEqual( get_tools_config('section', 'key', 'defaultVal'), 'defaultVal') @@ -30,10 +30,10 @@ class TestGuestCustUtil(CiTestCase): This test is designed to verify the behavior if internal exception is raised. """ - with mock.patch.object(util, 'which', return_value='/dummy/path'): - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'which', return_value='/dummy/path'): + with mock.patch.object(subp, 'subp', return_value=('key=value', b''), - side_effect=util.ProcessExecutionError( + side_effect=subp.ProcessExecutionError( "subp failed", exit_code=99)): # verify return value is 'defaultVal', not 'value'. self.assertEqual( @@ -45,28 +45,28 @@ class TestGuestCustUtil(CiTestCase): This test is designed to verify the value could be parsed from key = value of the given [section] """ - with mock.patch.object(util, 'which', return_value='/dummy/path'): + with mock.patch.object(subp, 'which', return_value='/dummy/path'): # value is not blank - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('key = value ', b'')): self.assertEqual( get_tools_config('section', 'key', 'defaultVal'), 'value') # value is blank - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('key = ', b'')): self.assertEqual( get_tools_config('section', 'key', 'defaultVal'), '') # value contains = - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('key=Bar=Wark', b'')): self.assertEqual( get_tools_config('section', 'key', 'defaultVal'), 'Bar=Wark') # value contains specific characters - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('[a] b.c_d=e-f', b'')): self.assertEqual( get_tools_config('section', 'key', 'defaultVal'), @@ -87,7 +87,7 @@ class TestGuestCustUtil(CiTestCase): # post gc status is YES, subp is called to execute command cf._insertKey("MISC|POST-GC-STATUS", "YES") conf = Config(cf) - with mock.patch.object(util, 'subp', + with mock.patch.object(subp, 'subp', return_value=('ok', b'')) as mockobj: self.assertEqual( set_gc_status(conf, 'Successful'), ('ok', b'')) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index c3113705..01916b70 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -4,5 +4,6 @@ dhensby lucasmoura matthewruffell nishigori +smoser tomponline TheRealFalcon -- cgit v1.2.3 From b36b61debe1556f6dfd2d7ff61fa7bd6b3335fa0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 8 Jun 2020 14:58:55 -0400 Subject: Move runparts to subp. (#420) runparts (run a directory of scripts) seems to fit well in subp module. The request to move it there was raised in #416. Replace use of logexc with LOG.debug as logexc comes from util. --- cloudinit/config/cc_scripts_per_boot.py | 4 ++-- cloudinit/config/cc_scripts_per_instance.py | 4 ++-- cloudinit/config/cc_scripts_per_once.py | 4 ++-- cloudinit/config/cc_scripts_user.py | 4 ++-- cloudinit/config/cc_scripts_vendor.py | 3 ++- cloudinit/subp.py | 32 +++++++++++++++++++++++++++++ cloudinit/util.py | 31 ---------------------------- 7 files changed, 42 insertions(+), 40 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py index 588e1b03..1e3f419e 100644 --- a/cloudinit/config/cc_scripts_per_boot.py +++ b/cloudinit/config/cc_scripts_per_boot.py @@ -24,7 +24,7 @@ module does not accept any config keys. import os -from cloudinit import util +from cloudinit import subp from cloudinit.settings import PER_ALWAYS @@ -38,7 +38,7 @@ def handle(name, _cfg, cloud, log, _args): # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: - util.runparts(runparts_path) + subp.runparts(runparts_path) except Exception: log.warning("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py index 75549b52..5966fb9a 100644 --- a/cloudinit/config/cc_scripts_per_instance.py +++ b/cloudinit/config/cc_scripts_per_instance.py @@ -27,7 +27,7 @@ the system. As a result per-instance scripts will run again. import os -from cloudinit import util +from cloudinit import subp from cloudinit.settings import PER_INSTANCE @@ -41,7 +41,7 @@ def handle(name, _cfg, cloud, log, _args): # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: - util.runparts(runparts_path) + subp.runparts(runparts_path) except Exception: log.warning("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py index 259bdfab..bcca859e 100644 --- a/cloudinit/config/cc_scripts_per_once.py +++ b/cloudinit/config/cc_scripts_per_once.py @@ -25,7 +25,7 @@ be run in alphabetical order. This module does not accept any config keys. import os -from cloudinit import util +from cloudinit import subp from cloudinit.settings import PER_ONCE @@ -39,7 +39,7 @@ def handle(name, _cfg, cloud, log, _args): # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: - util.runparts(runparts_path) + subp.runparts(runparts_path) except Exception: log.warning("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py index d940dbd6..215703ef 100644 --- a/cloudinit/config/cc_scripts_user.py +++ b/cloudinit/config/cc_scripts_user.py @@ -27,7 +27,7 @@ This module does not accept any config keys. import os -from cloudinit import util +from cloudinit import subp from cloudinit.settings import PER_INSTANCE @@ -42,7 +42,7 @@ def handle(name, _cfg, cloud, log, _args): # go here... runparts_path = os.path.join(cloud.get_ipath_cur(), SCRIPT_SUBDIR) try: - util.runparts(runparts_path) + subp.runparts(runparts_path) except Exception: log.warning("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) diff --git a/cloudinit/config/cc_scripts_vendor.py b/cloudinit/config/cc_scripts_vendor.py index faac9242..e0a4bfff 100644 --- a/cloudinit/config/cc_scripts_vendor.py +++ b/cloudinit/config/cc_scripts_vendor.py @@ -28,6 +28,7 @@ entry under the ``vendor_data`` config key. import os +from cloudinit import subp from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -46,7 +47,7 @@ def handle(name, cfg, cloud, log, _args): prefix = util.get_cfg_by_path(cfg, ('vendor_data', 'prefix'), []) try: - util.runparts(runparts_path, exe_prefix=prefix) + subp.runparts(runparts_path, exe_prefix=prefix) except Exception: log.warning("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) diff --git a/cloudinit/subp.py b/cloudinit/subp.py index f8400b1f..804ef3ca 100644 --- a/cloudinit/subp.py +++ b/cloudinit/subp.py @@ -351,4 +351,36 @@ def is_exe(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) +def runparts(dirp, skip_no_exist=True, exe_prefix=None): + if skip_no_exist and not os.path.isdir(dirp): + return + + failed = [] + attempted = [] + + if exe_prefix is None: + prefix = [] + elif isinstance(exe_prefix, str): + prefix = [str(exe_prefix)] + elif isinstance(exe_prefix, list): + prefix = exe_prefix + else: + raise TypeError("exe_prefix must be None, str, or list") + + for exe_name in sorted(os.listdir(dirp)): + exe_path = os.path.join(dirp, exe_name) + if is_exe(exe_path): + attempted.append(exe_path) + try: + subp(prefix + [exe_path], capture=False) + except ProcessExecutionError as e: + LOG.debug(e) + failed.append(exe_name) + + if failed and attempted: + raise RuntimeError( + 'Runparts: %s failures (%s) in %s attempted commands' % + (len(failed), ",".join(failed), len(attempted))) + + # vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index 445e3d4c..ab2df59e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -749,37 +749,6 @@ def del_dir(path): shutil.rmtree(path) -def runparts(dirp, skip_no_exist=True, exe_prefix=None): - if skip_no_exist and not os.path.isdir(dirp): - return - - failed = [] - attempted = [] - - if exe_prefix is None: - prefix = [] - elif isinstance(exe_prefix, str): - prefix = [str(exe_prefix)] - elif isinstance(exe_prefix, list): - prefix = exe_prefix - else: - raise TypeError("exe_prefix must be None, str, or list") - - for exe_name in sorted(os.listdir(dirp)): - exe_path = os.path.join(dirp, exe_name) - if os.path.isfile(exe_path) and os.access(exe_path, os.X_OK): - attempted.append(exe_path) - try: - subp.subp(prefix + [exe_path], capture=False) - except subp.ProcessExecutionError as e: - logexc(LOG, "Failed running %s [%s]", exe_path, e.exit_code) - failed.append(e) - - if failed and attempted: - raise RuntimeError('Runparts: %s failures in %s attempted commands' - % (len(failed), len(attempted))) - - # read_optional_seed # returns boolean indicating success or failure (presense of files) # if files are present, populates 'fill' dictionary with 'user-data' and -- cgit v1.2.3 From 4e534a7c22514d162e65cc773d44b706fe877cea Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 8 Jun 2020 15:43:24 -0400 Subject: Replace cc_chef is_installed with use of subp.is_exe. (#421) This was brought up in review of #416. Makes sense to remove the local copy of "is this executable file". --- cloudinit/config/cc_chef.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index e1f51fce..f8c22bcb 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -145,14 +145,6 @@ CHEF_EXEC_PATH = '/usr/bin/chef-client' CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) -def is_installed(): - if not os.path.isfile(CHEF_EXEC_PATH): - return False - if not os.access(CHEF_EXEC_PATH, os.X_OK): - return False - return True - - def post_run_chef(chef_cfg, log): delete_pem = util.get_cfg_option_bool(chef_cfg, 'delete_validation_post_exec', @@ -257,9 +249,10 @@ def handle(name, cfg, cloud, log, _args): # Try to install chef, if its not already installed... force_install = util.get_cfg_option_bool(chef_cfg, 'force_install', default=False) - if not is_installed() or force_install: + installed = subp.is_exe(CHEF_EXEC_PATH) + if not installed or force_install: run = install_chef(cloud, chef_cfg, log) - elif is_installed(): + elif installed: run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False) else: run = False -- cgit v1.2.3 From 0755cff078d5931e1d8e151bdcb84afb92bc0f02 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 22 Jun 2020 10:30:05 -0400 Subject: cc_final_message: don't create directories when writing boot-finished (#445) If the instance symlink doesn't exist, then we shouldn't create a directory in its place, because that breaks future boots. LP: #1883903 --- cloudinit/config/cc_final_message.py | 2 +- cloudinit/config/tests/test_final_message.py | 33 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 cloudinit/config/tests/test_final_message.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index fd141541..3441f7a9 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -78,7 +78,7 @@ def handle(_name, cfg, cloud, log, args): boot_fin_fn = cloud.paths.boot_finished try: contents = "%s - %s - v. %s\n" % (uptime, ts, cver) - util.write_file(boot_fin_fn, contents) + util.write_file(boot_fin_fn, contents, ensure_dir_exists=False) except Exception: util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn) diff --git a/cloudinit/config/tests/test_final_message.py b/cloudinit/config/tests/test_final_message.py new file mode 100644 index 00000000..76cb0ad1 --- /dev/null +++ b/cloudinit/config/tests/test_final_message.py @@ -0,0 +1,33 @@ +# This file is part of cloud-init. See LICENSE file for license information. +from unittest import mock + +import pytest + +from cloudinit.config.cc_final_message import handle + + +class TestHandle: + # TODO: Expand these tests to cover full functionality; currently they only + # cover the logic around how the boot-finished file is written (and not its + # contents). + + @pytest.mark.parametrize( + "instance_dir_exists,file_is_written", [(True, True), (False, False)] + ) + def test_boot_finished_written( + self, instance_dir_exists, file_is_written, tmpdir + ): + instance_dir = tmpdir.join("var/lib/cloud/instance") + if instance_dir_exists: + instance_dir.ensure_dir() + boot_finished = instance_dir.join("boot-finished") + + m_cloud = mock.Mock( + paths=mock.Mock(boot_finished=boot_finished.strpath) + ) + + handle(None, {}, m_cloud, mock.Mock(), []) + + # We should not change the status of the instance directory + assert instance_dir_exists == instance_dir.exists() + assert file_is_written == boot_finished.exists() -- cgit v1.2.3 From b0eba6a4c3a25a13259c01ac4c9a35720fecfdd2 Mon Sep 17 00:00:00 2001 From: lucasmoura Date: Mon, 29 Jun 2020 13:21:04 -0300 Subject: Add schema for cc_chef module (#375) Create a schema object for the chef module and validate this schema in the handle function of the module. Some of the config keys description, so I tried looking at the code and chef documentation to provide an information to the user. However, I don't know if I have the best description for all fields. For example, for the key show_time I could not find an accurate description of what it did, so I used what was in our code base to infer what it should do. LP: #1858888 --- cloudinit/config/cc_chef.py | 359 ++++++++++++++++++++++------ tests/unittests/test_handler/test_schema.py | 3 +- 2 files changed, 290 insertions(+), 72 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index f8c22bcb..aaf71366 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -6,81 +6,22 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Chef ----- -**Summary:** module that configures, starts and installs chef. - -This module enables chef to be installed (from packages or -from gems, or from omnibus). Before this occurs chef configurations are -written to disk (validation.pem, client.pem, firstboot.json, client.rb), -and needed chef folders/directories are created (/etc/chef and /var/log/chef -and so-on). Then once installing proceeds correctly if configured chef will -be started (in daemon mode or in non-daemon mode) and then once that has -finished (if ran in non-daemon mode this will be when chef finishes -converging, if ran in daemon mode then no further actions are possible since -chef will have forked into its own process) then a post run function can -run that can do finishing activities (such as removing the validation pem -file). - -**Internal name:** ``cc_chef`` - -**Module frequency:** per always - -**Supported distros:** all - -**Config keys**:: - - chef: - directories: (defaulting to /etc/chef, /var/log/chef, /var/lib/chef, - /var/cache/chef, /var/backups/chef, /var/run/chef) - validation_cert: (optional string to be written to file validation_key) - special value 'system' means set use existing file - validation_key: (optional the path for validation_cert. default - /etc/chef/validation.pem) - firstboot_path: (path to write run_list and initial_attributes keys that - should also be present in this configuration, defaults - to /etc/chef/firstboot.json) - exec: boolean to run or not run chef (defaults to false, unless - a gem installed is requested - where this will then default - to true) - - chef.rb template keys (if falsey, then will be skipped and not - written to /etc/chef/client.rb) - - chef: - chef_license: - client_key: - encrypted_data_bag_secret: - environment: - file_backup_path: - file_cache_path: - json_attribs: - log_level: - log_location: - node_name: - omnibus_url: - omnibus_url_retries: - omnibus_version: - pid_file: - server_url: - show_time: - ssl_verify_mode: - validation_cert: - validation_key: - validation_name: -""" +"""Chef: module that configures, starts and installs chef.""" import itertools import json import os +from textwrap import dedent from cloudinit import subp +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit import templater from cloudinit import temp_utils from cloudinit import url_helper from cloudinit import util +from cloudinit.settings import PER_ALWAYS + RUBY_VERSION_DEFAULT = "1.8" @@ -101,6 +42,8 @@ OMNIBUS_URL = "https://www.chef.io/chef/install.sh" OMNIBUS_URL_RETRIES = 5 CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' +CHEF_ENCRYPTED_DATA_BAG_PATH = '/etc/chef/encrypted_data_bag_secret' +CHEF_ENVIRONMENT = '_default' CHEF_FB_PATH = '/etc/chef/firstboot.json' CHEF_RB_TPL_DEFAULTS = { # These are ruby symbols... @@ -110,11 +53,11 @@ CHEF_RB_TPL_DEFAULTS = { 'log_location': '/var/log/chef/client.log', 'validation_key': CHEF_VALIDATION_PEM_PATH, 'validation_cert': None, - 'client_key': "/etc/chef/client.pem", + 'client_key': '/etc/chef/client.pem', 'json_attribs': CHEF_FB_PATH, - 'file_cache_path': "/var/cache/chef", - 'file_backup_path': "/var/backups/chef", - 'pid_file': "/var/run/chef/client.pid", + 'file_cache_path': '/var/cache/chef', + 'file_backup_path': '/var/backups/chef', + 'pid_file': '/var/run/chef/client.pid', 'show_time': True, 'encrypted_data_bag_secret': None, } @@ -125,7 +68,6 @@ CHEF_RB_TPL_PATH_KEYS = frozenset([ 'client_key', 'file_cache_path', 'json_attribs', - 'file_cache_path', 'pid_file', 'encrypted_data_bag_secret', 'chef_license', @@ -145,6 +87,279 @@ CHEF_EXEC_PATH = '/usr/bin/chef-client' CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) +frequency = PER_ALWAYS +distros = ["all"] +schema = { + 'id': 'cc_chef', + 'name': 'Chef', + 'title': 'module that configures, starts and installs chef', + 'description': dedent("""\ + This module enables chef to be installed (from packages, + gems, or from omnibus). Before this occurs, chef configuration is + written to disk (validation.pem, client.pem, firstboot.json, + client.rb), and required directories are created (/etc/chef and + /var/log/chef and so-on). If configured, chef will be + installed and started in either daemon or non-daemon mode. + If run in non-daemon mode, post run actions are executed to do + finishing activities such as removing validation.pem."""), + 'distros': distros, + 'examples': [dedent(""" + chef: + directories: + - /etc/chef + - /var/log/chef + validation_cert: system + install_type: omnibus + initial_attributes: + apache: + prefork: + maxclients: 100 + keepalive: off + run_list: + - recipe[apache2] + - role[db] + encrypted_data_bag_secret: /etc/chef/encrypted_data_bag_secret + environment: _default + log_level: :auto + omnibus_url_retries: 2 + server_url: https://chef.yourorg.com:4000 + ssl_verify_mode: :verify_peer + validation_name: yourorg-validator""")], + 'frequency': frequency, + 'type': 'object', + 'properties': { + 'chef': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'directories': { + 'type': 'array', + 'items': { + 'type': 'string' + }, + 'uniqueItems': True, + 'description': dedent("""\ + Create the necessary directories for chef to run. By + default, it creates the following directories: + + {chef_dirs}""").format( + chef_dirs="\n".join( + [" - ``{}``".format(d) for d in CHEF_DIRS] + ) + ) + }, + 'validation_cert': { + 'type': 'string', + 'description': dedent("""\ + Optional string to be written to file validation_key. + Special value ``system`` means set use existing file. + """) + }, + 'validation_key': { + 'type': 'string', + 'default': CHEF_VALIDATION_PEM_PATH, + 'description': dedent("""\ + Optional path for validation_cert. default to + ``{}``.""".format(CHEF_VALIDATION_PEM_PATH)) + }, + 'firstboot_path': { + 'type': 'string', + 'default': CHEF_FB_PATH, + 'description': dedent("""\ + Path to write run_list and initial_attributes keys that + should also be present in this configuration, defaults + to ``{}``.""".format(CHEF_FB_PATH)) + }, + 'exec': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + define if we should run or not run chef (defaults to + false, unless a gem installed is requested where this + will then default to true).""") + }, + 'client_key': { + 'type': 'string', + 'default': CHEF_RB_TPL_DEFAULTS['client_key'], + 'description': dedent("""\ + Optional path for client_cert. default to + ``{}``.""".format(CHEF_RB_TPL_DEFAULTS['client_key'])) + }, + 'encrypted_data_bag_secret': { + 'type': 'string', + 'default': None, + 'description': dedent("""\ + Specifies the location of the secret key used by chef + to encrypt data items. By default, this path is set + to None, meaning that chef will have to look at the + path ``{}`` for it. + """.format(CHEF_ENCRYPTED_DATA_BAG_PATH)) + }, + 'environment': { + 'type': 'string', + 'default': CHEF_ENVIRONMENT, + 'description': dedent("""\ + Specifies which environment chef will use. By default, + it will use the ``{}`` configuration. + """.format(CHEF_ENVIRONMENT)) + }, + 'file_backup_path': { + 'type': 'string', + 'default': CHEF_RB_TPL_DEFAULTS['file_backup_path'], + 'description': dedent("""\ + Specifies the location in which backup files are + stored. By default, it uses the + ``{}`` location.""".format( + CHEF_RB_TPL_DEFAULTS['file_backup_path'])) + }, + 'file_cache_path': { + 'type': 'string', + 'default': CHEF_RB_TPL_DEFAULTS['file_cache_path'], + 'description': dedent("""\ + Specifies the location in which chef cache files will + be saved. By default, it uses the ``{}`` + location.""".format( + CHEF_RB_TPL_DEFAULTS['file_cache_path'])) + }, + 'json_attribs': { + 'type': 'string', + 'default': CHEF_FB_PATH, + 'description': dedent("""\ + Specifies the location in which some chef json data is + stored. By default, it uses the + ``{}`` location.""".format(CHEF_FB_PATH)) + }, + 'log_level': { + 'type': 'string', + 'default': CHEF_RB_TPL_DEFAULTS['log_level'], + 'description': dedent("""\ + Defines the level of logging to be stored in the log + file. By default this value is set to ``{}``. + """.format(CHEF_RB_TPL_DEFAULTS['log_level'])) + }, + 'log_location': { + 'type': 'string', + 'default': CHEF_RB_TPL_DEFAULTS['log_location'], + 'description': dedent("""\ + Specifies the location of the chef lof file. By + default, the location is specified at + ``{}``.""".format( + CHEF_RB_TPL_DEFAULTS['log_location'])) + }, + 'node_name': { + 'type': 'string', + 'description': dedent("""\ + The name of the node to run. By default, we will + use th instance id as the node name.""") + }, + 'omnibus_url': { + 'type': 'string', + 'default': OMNIBUS_URL, + 'description': dedent("""\ + Omnibus URL if chef should be installed through + Omnibus. By default, it uses the + ``{}``.""".format(OMNIBUS_URL)) + }, + 'omnibus_url_retries': { + 'type': 'integer', + 'default': OMNIBUS_URL_RETRIES, + 'description': dedent("""\ + The number of retries that will be attempted to reach + the Omnibus URL""") + }, + 'omnibus_version': { + 'type': 'string', + 'description': dedent("""\ + Optional version string to require for omnibus + install.""") + }, + 'pid_file': { + 'type': 'string', + 'default': CHEF_RB_TPL_DEFAULTS['pid_file'], + 'description': dedent("""\ + The location in which a process identification + number (pid) is saved. By default, it saves + in the ``{}`` location.""".format( + CHEF_RB_TPL_DEFAULTS['pid_file'])) + }, + 'server_url': { + 'type': 'string', + 'description': 'The URL for the chef server' + }, + 'show_time': { + 'type': 'boolean', + 'default': True, + 'description': 'Show time in chef logs' + }, + 'ssl_verify_mode': { + 'type': 'string', + 'default': CHEF_RB_TPL_DEFAULTS['ssl_verify_mode'], + 'description': dedent("""\ + Set the verify mode for HTTPS requests. We can have + two possible values for this parameter: + + - ``:verify_none``: No validation of SSL \ + certificates. + - ``:verify_peer``: Validate all SSL certificates. + + By default, the parameter is set as ``{}``. + """.format(CHEF_RB_TPL_DEFAULTS['ssl_verify_mode'])) + }, + 'validation_name': { + 'type': 'string', + 'description': dedent("""\ + The name of the chef-validator key that Chef Infra + Client uses to access the Chef Infra Server during + the initial Chef Infra Client run.""") + }, + 'force_install': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + If set to ``True``, forces chef installation, even + if it is already installed.""") + }, + 'initial_attributes': { + 'type': 'object', + 'items': { + 'type': 'string' + }, + 'description': dedent("""\ + Specify a list of initial attributes used by the + cookbooks.""") + }, + 'install_type': { + 'type': 'string', + 'default': 'packages', + 'description': dedent("""\ + The type of installation for chef. It can be one of + the following values: + + - ``packages`` + - ``gems`` + - ``omnibus``""") + }, + 'run_list': { + 'type': 'array', + 'items': { + 'type': 'string' + }, + 'description': 'A run list for a first boot json.' + }, + "chef_license": { + 'type': 'string', + 'description': dedent("""\ + string that indicates if user accepts or not license + related to some of chef products""") + } + } + } + } +} + +__doc__ = get_schema_doc(schema) + + def post_run_chef(chef_cfg, log): delete_pem = util.get_cfg_option_bool(chef_cfg, 'delete_validation_post_exec', @@ -192,6 +407,8 @@ def handle(name, cfg, cloud, log, _args): log.debug(("Skipping module named %s," " no 'chef' key in configuration"), name) return + + validate_cloudconfig_schema(cfg, schema) chef_cfg = cfg['chef'] # Ensure the chef directories we use exist @@ -219,7 +436,7 @@ def handle(name, cfg, cloud, log, _args): iid = str(cloud.datasource.get_instance_id()) params = get_template_params(iid, chef_cfg, log) # Do a best effort attempt to ensure that the template values that - # are associated with paths have there parent directory created + # are associated with paths have their parent directory created # before they are used by the chef-client itself. param_paths = set() for (k, v) in params.items(): diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index e19d13b8..99f0b06c 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -34,7 +34,8 @@ class GetSchemaTest(CiTestCase): 'cc_ubuntu_advantage', 'cc_ubuntu_drivers', 'cc_write_files', - 'cc_zypper_add_repo' + 'cc_zypper_add_repo', + 'cc_chef' ], [subschema['id'] for subschema in schema['allOf']]) self.assertEqual('cloud-config-schema', schema['id']) -- cgit v1.2.3 From 66e114a660c53400e389f119781f378311b65108 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 30 Jun 2020 13:17:40 -0400 Subject: Enable use of the caplog fixture in pytest tests, and add a cc_final_message test using it (#461) caplog is only available in pytest itself from 3.0 onwards. In xenial, we only have pytest 2.8.7. However, in xenial we do have pytest-catchlog available (as python3-pytest-catchlog), so we use that where appropriate. --- .travis.yml | 2 +- HACKING.rst | 9 +++++---- cloudinit/config/tests/test_final_message.py | 19 ++++++++++++++++--- packages/bddeb | 2 ++ tox.ini | 1 + 5 files changed, 25 insertions(+), 8 deletions(-) (limited to 'cloudinit/config') diff --git a/.travis.yml b/.travis.yml index 95c591fd..9844c065 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,7 +72,7 @@ matrix: - cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc script: # Ubuntu LTS: Build - - ./packages/bddeb -S -d + - ./packages/bddeb -S -d --release xenial - | needs_caching=false if [ -e "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" ]; then diff --git a/HACKING.rst b/HACKING.rst index 1e8aca7a..27a38bc3 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -218,11 +218,12 @@ The following guidelines should be followed: [#fixture-list]_: * ``cache`` - * ``capsys`` * ``capfd`` - * ``record_xml_property`` + * ``caplog`` (provided by ``python3-pytest-catchlog`` on xenial) + * ``capsys`` * ``monkeypatch`` * ``pytestconfig`` + * ``record_xml_property`` * ``recwarn`` * ``tmpdir_factory`` * ``tmpdir`` @@ -328,9 +329,9 @@ variable annotations specified in `PEP-526`_ were introduced in Python .. [#fixture-list] This list of fixtures (with markup) can be reproduced by running:: - py.test-3 --fixtures -q | grep "^[^ ]" | grep -v no | sed 's/.*/* ``\0``/' + py.test-3 --fixtures -q | grep "^[^ -]" | grep -v '\(no\|capturelog\)' | sort | sed 's/.*/* ``\0``/' - in a xenial lxd container with python3-pytest installed. + in a xenial lxd container with python3-pytest-catchlog installed. Feature Flags ------------- diff --git a/cloudinit/config/tests/test_final_message.py b/cloudinit/config/tests/test_final_message.py index 76cb0ad1..46ba99b2 100644 --- a/cloudinit/config/tests/test_final_message.py +++ b/cloudinit/config/tests/test_final_message.py @@ -1,4 +1,5 @@ # This file is part of cloud-init. See LICENSE file for license information. +import logging from unittest import mock import pytest @@ -12,10 +13,19 @@ class TestHandle: # contents). @pytest.mark.parametrize( - "instance_dir_exists,file_is_written", [(True, True), (False, False)] + "instance_dir_exists,file_is_written,expected_log_substring", + [ + (True, True, None), + (False, False, "Failed to write boot finished file "), + ], ) def test_boot_finished_written( - self, instance_dir_exists, file_is_written, tmpdir + self, + instance_dir_exists, + file_is_written, + expected_log_substring, + caplog, + tmpdir, ): instance_dir = tmpdir.join("var/lib/cloud/instance") if instance_dir_exists: @@ -26,8 +36,11 @@ class TestHandle: paths=mock.Mock(boot_finished=boot_finished.strpath) ) - handle(None, {}, m_cloud, mock.Mock(), []) + handle(None, {}, m_cloud, logging.getLogger(), []) # We should not change the status of the instance directory assert instance_dir_exists == instance_dir.exists() assert file_is_written == boot_finished.exists() + + if expected_log_substring: + assert expected_log_substring in caplog.text diff --git a/packages/bddeb b/packages/bddeb index 78b1c83b..b0f219b6 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -91,6 +91,8 @@ def write_debian_folder(root, templ_data, cloud_util_deps): # NOTE: python package was moved to the front after debuild -S would fail with # 'Please add apropriate interpreter' errors (as in debian bug 861132) requires.extend(['python3'] + reqs + test_reqs) + if templ_data['debian_release'] == 'xenial': + requires.append('python3-pytest-catchlog') templater.render_to_file(util.abs_join(find_root(), 'packages', 'debian', 'control.in'), util.abs_join(deb_dir, 'control'), diff --git a/tox.ini b/tox.ini index ebcebc41..847a2fbd 100644 --- a/tox.ini +++ b/tox.ini @@ -79,6 +79,7 @@ deps = # test-requirements httpretty==0.9.6 mock==1.3.0 + pytest-catchlog==1.2.1 [testenv:xenial] # When updating this commands definition, also update the definition in -- cgit v1.2.3 From 3d06abc2e0017436dadbf9c26fefa9f95368db9a Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Fri, 10 Jul 2020 08:08:44 -0500 Subject: cc_mounts: handle missing fstab (#484) Do not fail if /etc/fstab is not present. Some images, like container rootfs may not include this file by default. LP: #1886531 --- cloudinit/config/cc_mounts.py | 21 +++++++++++---------- tests/unittests/test_handler/test_handler_mounts.py | 12 ++++++++++++ 2 files changed, 23 insertions(+), 10 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index e57d1b1f..773b8285 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -379,17 +379,18 @@ def handle(_name, cfg, cloud, log, _args): fstab_devs = {} fstab_removed = [] - for line in util.load_file(FSTAB_PATH).splitlines(): - if MNT_COMMENT in line: - fstab_removed.append(line) - continue + if os.path.exists(FSTAB_PATH): + for line in util.load_file(FSTAB_PATH).splitlines(): + if MNT_COMMENT in line: + fstab_removed.append(line) + continue - try: - toks = WS.split(line) - except Exception: - pass - fstab_devs[toks[0]] = line - fstab_lines.append(line) + try: + toks = WS.split(line) + except Exception: + pass + fstab_devs[toks[0]] = line + fstab_lines.append(line) for i in range(len(cfgmnt)): # skip something that wasn't a list diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index b643e3ae..e193f4d4 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -183,6 +183,18 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase): return dev + def test_no_fstab(self): + """ Handle images which do not include an fstab. """ + self.assertFalse(os.path.exists(cc_mounts.FSTAB_PATH)) + fstab_expected_content = ( + '%s\tnone\tswap\tsw,comment=cloudconfig\t' + '0\t0\n' % (self.swap_path,) + ) + cc_mounts.handle(None, {}, self.mock_cloud, self.mock_log, []) + with open(cc_mounts.FSTAB_PATH, 'r') as fd: + fstab_new_content = fd.read() + self.assertEqual(fstab_expected_content, fstab_new_content) + def test_swap_integrity(self): '''Ensure that the swap file is correctly created and can swapon successfully. Fixing the corner case of: -- cgit v1.2.3 From fecbd81889011e8a75badd18935297f3494fe485 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 13 Jul 2020 11:37:37 -0400 Subject: Fix two minor warnings (#475) --- cloudinit/config/cc_puppet.py | 4 ++-- tests/unittests/test_datasource/test_gce.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 635c73bc..bc981cf4 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -160,9 +160,9 @@ def handle(name, cfg, cloud, log, _args): cleaned_lines = [i.lstrip() for i in contents.splitlines()] cleaned_contents = '\n'.join(cleaned_lines) # Move to puppet_config.read_file when dropping py2.7 - puppet_config.readfp( # pylint: disable=W1505 + puppet_config.read_file( StringIO(cleaned_contents), - filename=p_constants.conf_path) + source=p_constants.conf_path) for (cfg_name, cfg) in puppet_cfg['conf'].items(): # Cert configuration is a special case # Dump the puppet master ca certificate in the correct place diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 4afbccff..01f4cbd1 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -114,7 +114,8 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase): self.assertTrue(success) req_header = httpretty.last_request().headers - self.assertDictContainsSubset(HEADERS, req_header) + for header_name, expected_value in HEADERS.items(): + self.assertEqual(expected_value, req_header.get(header_name)) def test_metadata(self): # UnicodeDecodeError if set to ds.userdata instead of userdata_raw -- cgit v1.2.3 From 3cec3881062490727c5fff1b16b53f0176f976f0 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 13 Jul 2020 12:00:32 -0400 Subject: cloudinit: remove global disable of pylint W0105 and fix errors (#480) This includes a fix to a test that had a string concatenation issue, and so was only testing a prefix of what was intended. --- .pylintrc | 3 +- cloudinit/analyze/show.py | 41 ++++++++++------------ cloudinit/config/cc_resizefs.py | 14 ++++---- cloudinit/distros/opensuse.py | 2 +- cloudinit/net/eni.py | 6 ++-- cloudinit/sources/DataSourceAzure.py | 14 ++++---- cloudinit/sources/DataSourceHetzner.py | 6 ++-- .../unittests/test_datasource/test_azure_helper.py | 7 ++-- tests/unittests/test_reporting_hyperv.py | 6 ++-- 9 files changed, 45 insertions(+), 54 deletions(-) (limited to 'cloudinit/config') diff --git a/.pylintrc b/.pylintrc index 4d5d066d..f04603b9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -7,7 +7,6 @@ jobs=4 [MESSAGES CONTROL] # Errors and warings with some filtered: -# W0105(pointless-string-statement) # W0107(unnecessary-pass) # W0201(attribute-defined-outside-init) # W0212(protected-access) @@ -28,7 +27,7 @@ jobs=4 # W0703(broad-except) # W1401(anomalous-backslash-in-string) -disable=C, F, I, R, W0105, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401 +disable=C, F, I, R, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401 [REPORTS] diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py index cca1fa7f..0c825b23 100644 --- a/cloudinit/analyze/show.py +++ b/cloudinit/analyze/show.py @@ -15,28 +15,25 @@ from cloudinit import subp from cloudinit import util from cloudinit.distros import uses_systemd -# An event: -''' -{ - "description": "executing late commands", - "event_type": "start", - "level": "INFO", - "name": "cmd-install/stage-late" - "origin": "cloudinit", - "timestamp": 1461164249.1590767, -}, - - { - "description": "executing late commands", - "event_type": "finish", - "level": "INFO", - "name": "cmd-install/stage-late", - "origin": "cloudinit", - "result": "SUCCESS", - "timestamp": 1461164249.1590767 - } - -''' +# Example events: +# { +# "description": "executing late commands", +# "event_type": "start", +# "level": "INFO", +# "name": "cmd-install/stage-late" +# "origin": "cloudinit", +# "timestamp": 1461164249.1590767, +# } +# { +# "description": "executing late commands", +# "event_type": "finish", +# "level": "INFO", +# "name": "cmd-install/stage-late", +# "origin": "cloudinit", +# "result": "SUCCESS", +# "timestamp": 1461164249.1590767 +# } + format_key = { '%d': 'delta', '%D': 'description', diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 8de4db30..978d2ee0 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -118,14 +118,12 @@ def _can_skip_resize_ufs(mount_point, devpth): if o == "-f": frag_sz = int(a) # check the current partition size - """ - # gpart show /dev/da0 -=> 40 62914480 da0 GPT (30G) - 40 1024 1 freebsd-boot (512K) - 1064 58719232 2 freebsd-ufs (28G) - 58720296 3145728 3 freebsd-swap (1.5G) - 61866024 1048496 - free - (512M) - """ + # Example output from `gpart show /dev/da0`: + # => 40 62914480 da0 GPT (30G) + # 40 1024 1 freebsd-boot (512K) + # 1064 58719232 2 freebsd-ufs (28G) + # 58720296 3145728 3 freebsd-swap (1.5G) + # 61866024 1048496 - free - (512M) expect_sz = None m = re.search('^(/dev/.+)p([0-9])$', devpth) gpart_res = _get_gpart_output(m.group(1)) diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py index ffb7d0e8..b8e557b8 100644 --- a/cloudinit/distros/opensuse.py +++ b/cloudinit/distros/opensuse.py @@ -185,7 +185,7 @@ class Distro(distros.Distro): def preferred_ntp_clients(self): """The preferred ntp client is dependent on the version.""" - """Allow distro to determine the preferred ntp client list""" + # Allow distro to determine the preferred ntp client list if not self._preferred_ntp_clients: distro_info = util.system_info()['dist'] name = distro_info[0] diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index b4c69457..13c041f3 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -483,10 +483,8 @@ class Renderer(renderer.Renderer): if searchdomains: lo['subnets'][0]["dns_search"] = (" ".join(searchdomains)) - ''' Apply a sort order to ensure that we write out - the physical interfaces first; this is critical for - bonding - ''' + # Apply a sort order to ensure that we write out the physical + # interfaces first; this is critical for bonding order = { 'loopback': 0, 'physical': 1, diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 41431a7a..5e25b956 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -166,12 +166,11 @@ def get_resource_disk_on_freebsd(port_id): port_id = port_id - 2 g1 = "000" + str(port_id) g0g1 = "{0}-{1}".format(g0, g1) - """ - search 'X' from - 'dev.storvsc.X.%pnpinfo: - classid=32412632-86cb-44a2-9b5c-50d1417354f5 - deviceid=00000000-0001-8899-0000-000000000000' - """ + + # search 'X' from + # 'dev.storvsc.X.%pnpinfo: + # classid=32412632-86cb-44a2-9b5c-50d1417354f5 + # deviceid=00000000-0001-8899-0000-000000000000' sysctl_out = get_dev_storvsc_sysctl() storvscid = find_storvscid_from_sysctl_pnpinfo(sysctl_out, g0g1) @@ -1485,13 +1484,12 @@ def maybe_remove_ubuntu_network_config_scripts(paths=None): def _is_platform_viable(seed_dir): + """Check platform environment to report if this datasource may run.""" with events.ReportEventStack( name="check-platform-viability", description="found azure asset tag", parent=azure_ds_reporter ) as evt: - - """Check platform environment to report if this datasource may run.""" asset_tag = util.read_dmi_data('chassis-asset-tag') if asset_tag == AZURE_CHASSIS_ASSET_TAG: return True diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py index 70e4274c..a86035e0 100644 --- a/cloudinit/sources/DataSourceHetzner.py +++ b/cloudinit/sources/DataSourceHetzner.py @@ -69,9 +69,9 @@ class DataSourceHetzner(sources.DataSource): self.userdata_raw = hc_helper.maybe_b64decode(ud) self.metadata_full = md - """hostname is name provided by user at launch. The API enforces - it is a valid hostname, but it is not guaranteed to be resolvable - in dns or fully qualified.""" + # hostname is name provided by user at launch. The API enforces it is + # a valid hostname, but it is not guaranteed to be resolvable in dns or + # fully qualified. self.metadata['instance-id'] = md['instance-id'] self.metadata['local-hostname'] = md['hostname'] self.metadata['network-config'] = md.get('network-config', None) diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 71ef57f0..f314cd4a 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -404,11 +404,10 @@ class TestWALinuxAgentShim(CiTestCase): self.GoalState.call_args_list) def test_certificates_used_to_determine_public_keys(self): + # if register_with_azure_and_fetch_data() isn't passed some info about + # the user's public keys, there's no point in even trying to parse the + # certificates shim = wa_shim() - """if register_with_azure_and_fetch_data() isn't passed some info about - the user's public keys, there's no point in even trying to parse - the certificates - """ mypk = [{'fingerprint': 'fp1', 'path': 'path1'}, {'fingerprint': 'fp3', 'path': 'path3', 'value': ''}] certs = {'fp1': 'expected-key', diff --git a/tests/unittests/test_reporting_hyperv.py b/tests/unittests/test_reporting_hyperv.py index b60a66ab..bacf5da9 100644 --- a/tests/unittests/test_reporting_hyperv.py +++ b/tests/unittests/test_reporting_hyperv.py @@ -139,8 +139,10 @@ class TextKvpReporter(CiTestCase): @mock.patch('cloudinit.subp.subp') def test_get_boot_telemetry(self, m_subp, m_sysd): reporter = HyperVKvpReportingHandler(kvp_file_path=self.tmp_file_path) - datetime_pattern = r"\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]" - r"\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z)" + datetime_pattern = ( + r"\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]" + r"\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z)" + ) # get_boot_telemetry makes two subp calls to systemctl. We provide # a list of values that the subp calls should return -- cgit v1.2.3 From 4fe576516d65feda17ba78e9265a8e494a195e7b Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 15 Jul 2020 10:26:12 -0400 Subject: cloudinit: remove global disable of pylint W0107 and fix errors (#489) * cloudinit: remove global disable of pylint W0107 and fix errors This includes removing a test class which contained no tests but wasn't detected as empty because of an errant pass statement. * .pylintrc: update disable comment to match arguments --- .pylintrc | 4 +--- cloudinit/config/cc_set_hostname.py | 1 - cloudinit/distros/networking.py | 3 --- cloudinit/distros/ubuntu.py | 2 -- cloudinit/net/cmdline.py | 2 -- cloudinit/net/dhcp.py | 2 -- cloudinit/reporting/handlers.py | 1 - cloudinit/sources/DataSourceAzure.py | 1 - cloudinit/sources/__init__.py | 2 -- cloudinit/sources/helpers/netlink.py | 1 - cloudinit/sources/helpers/vmware/imc/config_file.py | 1 - cloudinit/sources/helpers/vmware/imc/config_namespace.py | 1 - cloudinit/sources/helpers/vmware/imc/config_source.py | 1 - cloudinit/stages.py | 1 - tests/cloud_tests/platforms/azurecloud/instance.py | 2 -- tests/cloud_tests/platforms/images.py | 1 - tests/cloud_tests/platforms/snapshots.py | 1 - tests/cloud_tests/testcases/base.py | 1 - tests/unittests/test_datasource/test_ibmcloud.py | 7 ------- tests/unittests/test_reporting.py | 1 - 20 files changed, 1 insertion(+), 35 deletions(-) (limited to 'cloudinit/config') diff --git a/.pylintrc b/.pylintrc index f04603b9..94a81d0e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -7,7 +7,6 @@ jobs=4 [MESSAGES CONTROL] # Errors and warings with some filtered: -# W0107(unnecessary-pass) # W0201(attribute-defined-outside-init) # W0212(protected-access) # W0221(arguments-differ) @@ -19,7 +18,6 @@ jobs=4 # W0602(global-variable-not-assigned) # W0603(global-statement) # W0611(unused-import) -# W0612(unused-variable) # W0613(unused-argument) # W0621(redefined-outer-name) # W0622(redefined-builtin) @@ -27,7 +25,7 @@ jobs=4 # W0703(broad-except) # W1401(anomalous-backslash-in-string) -disable=C, F, I, R, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401 +disable=C, F, I, R, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401 [REPORTS] diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index 10d6d197..c13020d8 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -55,7 +55,6 @@ class SetHostnameError(Exception): This may happen if we attempt to set the hostname early in cloud-init's init-local timeframe as certain services may not be running yet. """ - pass def handle(name, cfg, cloud, log, _args): diff --git a/cloudinit/distros/networking.py b/cloudinit/distros/networking.py index 75e69df5..10ed249d 100644 --- a/cloudinit/distros/networking.py +++ b/cloudinit/distros/networking.py @@ -92,7 +92,6 @@ class Networking(metaclass=abc.ABCMeta): Examples of non-physical network devices: bonds, bridges, tunnels, loopback devices. """ - pass def is_renamed(self, devname: DeviceName) -> bool: return net.is_renamed(devname) @@ -117,7 +116,6 @@ class Networking(metaclass=abc.ABCMeta): process entirely, if the device already exists.) :type exists: Optional[DeviceName] """ - pass def wait_for_physdevs( self, netcfg: NetworkConfig, *, strict: bool = True @@ -182,7 +180,6 @@ class BSDNetworking(Networking): def settle(self, *, exists=None) -> None: """BSD has no equivalent to `udevadm settle`; noop.""" - pass class LinuxNetworking(Networking): diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index 23be3bdd..b4c4b0c3 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -49,7 +49,5 @@ class Distro(debian.Distro): copy.deepcopy(PREFERRED_NTP_CLIENTS)) return self._preferred_ntp_clients - pass - # vi: ts=4 expandtab diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 0e83685d..7ca7262b 100755 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -29,12 +29,10 @@ class InitramfsNetworkConfigSource(metaclass=abc.ABCMeta): @abc.abstractmethod def is_applicable(self) -> bool: """Is this initramfs config source applicable to the current system?""" - pass @abc.abstractmethod def render_config(self) -> dict: """Render a v1 network config from the initramfs configuration""" - pass class KlibcNetworkConfigSource(InitramfsNetworkConfigSource): diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 9e004688..d1d1255e 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -31,12 +31,10 @@ class InvalidDHCPLeaseFileError(Exception): Current uses are DataSourceAzure and DataSourceEc2 during ephemeral boot to scrape metadata. """ - pass class NoDHCPLeaseError(Exception): """Raised when unable to get a DHCP lease.""" - pass class EphemeralDHCPv4(object): diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 6b9127b6..1986ebdd 100755 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -35,7 +35,6 @@ class ReportingHandler(metaclass=abc.ABCMeta): def flush(self): """Ensure ReportingHandler has published all events""" - pass class LogHandler(ReportingHandler): diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 5e25b956..8b34d047 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -688,7 +688,6 @@ class DataSourceAzure(sources.DataSource): except UrlError: # Teardown our EphemeralDHCPv4 context on failure as we retry self._ephemeral_dhcp_ctx.clean_network() - pass finally: if nl_sock: nl_sock.close() diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 923e3cea..c4d60fff 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -78,7 +78,6 @@ class DataSourceNotFoundException(Exception): class InvalidMetaDataException(Exception): """Raised when metadata is broken, unavailable or disabled.""" - pass def process_instance_metadata(metadata, key_path='', sensitive_keys=()): @@ -511,7 +510,6 @@ class DataSource(metaclass=abc.ABCMeta): (e.g. 'ssh-rsa') and key_value is the key itself (e.g. 'AAAAB3NzaC1y...'). """ - pass def _remap_device(self, short_name): # LP: #611137 diff --git a/cloudinit/sources/helpers/netlink.py b/cloudinit/sources/helpers/netlink.py index d377ae3d..a74a3588 100644 --- a/cloudinit/sources/helpers/netlink.py +++ b/cloudinit/sources/helpers/netlink.py @@ -55,7 +55,6 @@ NetlinkHeader = namedtuple('NetlinkHeader', ['length', 'type', 'flags', 'seq', class NetlinkCreateSocketError(RuntimeError): '''Raised if netlink socket fails during create or bind.''' - pass def create_bound_netlink_socket(): diff --git a/cloudinit/sources/helpers/vmware/imc/config_file.py b/cloudinit/sources/helpers/vmware/imc/config_file.py index 602af078..fc034c95 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_file.py +++ b/cloudinit/sources/helpers/vmware/imc/config_file.py @@ -22,7 +22,6 @@ class ConfigFile(ConfigSource, dict): def __init__(self, filename): self._loadConfigFile(filename) - pass def _insertKey(self, key, val): """ diff --git a/cloudinit/sources/helpers/vmware/imc/config_namespace.py b/cloudinit/sources/helpers/vmware/imc/config_namespace.py index 2f29edd4..5899d8f7 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_namespace.py +++ b/cloudinit/sources/helpers/vmware/imc/config_namespace.py @@ -10,6 +10,5 @@ from .config_source import ConfigSource class ConfigNamespace(ConfigSource): """Specifies the Config Namespace.""" - pass # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py index 2f8ea546..7ec06a9c 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_source.py +++ b/cloudinit/sources/helpers/vmware/imc/config_source.py @@ -8,6 +8,5 @@ class ConfigSource(object): """Specifies a source for the Config Content.""" - pass # vi: ts=4 expandtab diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 69e6b7e1..765f4aab 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -947,7 +947,6 @@ def _pkl_load(fname): except Exception as e: if os.path.isfile(fname): LOG.warning("failed loading pickle in %s: %s", fname, e) - pass # This is allowed so just return nothing successfully loaded... if not pickle_contents: diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py index f1e28a96..a136cf0d 100644 --- a/tests/cloud_tests/platforms/azurecloud/instance.py +++ b/tests/cloud_tests/platforms/azurecloud/instance.py @@ -80,7 +80,6 @@ class AzureCloudInstance(Instance): except CloudError: LOG.debug(('image not found, launching instance with base image, ' 'image_id=%s'), self.image_id) - pass vm_params = { 'name': self.vm_name, @@ -169,7 +168,6 @@ class AzureCloudInstance(Instance): sleep(15) else: LOG.warning('Could not find console log: %s', e) - pass LOG.debug('stopping instance %s', self.image_id) vm_deallocate = \ diff --git a/tests/cloud_tests/platforms/images.py b/tests/cloud_tests/platforms/images.py index 557a5cf6..f047de2e 100644 --- a/tests/cloud_tests/platforms/images.py +++ b/tests/cloud_tests/platforms/images.py @@ -52,6 +52,5 @@ class Image(TargetBase): def destroy(self): """Clean up data associated with image.""" - pass # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/snapshots.py b/tests/cloud_tests/platforms/snapshots.py index 94328982..0f5f8bb6 100644 --- a/tests/cloud_tests/platforms/snapshots.py +++ b/tests/cloud_tests/platforms/snapshots.py @@ -40,6 +40,5 @@ class Snapshot(object): def destroy(self): """Clean up snapshot data.""" - pass # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 2e7c6686..4448e0b5 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -34,7 +34,6 @@ class CloudTestCase(unittest.TestCase): @classmethod def maybeSkipTest(cls): """Present to allow subclasses to override and raise a skipTest.""" - pass def assertPackageInstalled(self, name, version=None): """Check dpkg-query --show output for matching package name. diff --git a/tests/unittests/test_datasource/test_ibmcloud.py b/tests/unittests/test_datasource/test_ibmcloud.py index 0b54f585..9013ae9f 100644 --- a/tests/unittests/test_datasource/test_ibmcloud.py +++ b/tests/unittests/test_datasource/test_ibmcloud.py @@ -15,13 +15,6 @@ mock = test_helpers.mock D_PATH = "cloudinit.sources.DataSourceIBMCloud." -class TestIBMCloud(test_helpers.CiTestCase): - """Test the datasource.""" - def setUp(self): - super(TestIBMCloud, self).setUp() - pass - - @mock.patch(D_PATH + "_is_xen", return_value=True) @mock.patch(D_PATH + "_is_ibm_provisioning") @mock.patch(D_PATH + "util.blkid") diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 6814030e..9f11fd5c 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -349,7 +349,6 @@ class TestReportingEventStack(TestCase): with parent: with child: pass - pass self.assertEqual(report_start.call_count, 0) self.assertEqual(report_finish.call_count, 0) -- cgit v1.2.3 From c0d239c7eadeae9c9e0b4e692b49e79c8ffcafd2 Mon Sep 17 00:00:00 2001 From: dermotbradley Date: Wed, 15 Jul 2020 21:24:43 +0100 Subject: cc_ca_certs.py: fix blank line problem when removing CAs and adding new one (#483) Problem: When cc_ca_certs configuration has both "remove-defaults: true" and also specifies one, or more, new trusted CAs to add then the resultant /etc/ca-certificates.conf file's 1st line is blank. As noted in comments in the existing cc_ca_certs.py code blank lines in this file cause problems. Fix: Before adding the cloud-init CA filename to this file first check the size of the file - if is is empty (as all existing CAs have been deleted) then write only the cloud-init CA filename to the file rather than appending it to the file. --- cloudinit/config/cc_ca_certs.py | 20 +++++++++++++------- .../test_handler/test_handler_ca_certs.py | 22 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 7617a8ea..910b78de 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -67,13 +67,19 @@ def add_ca_certs(certs): cert_file_contents = "\n".join([str(c) for c in certs]) util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0o644) - # Append cert filename to CA_CERT_CONFIG file. - # We have to strip the content because blank lines in the file - # causes subsequent entries to be ignored. (LP: #1077020) - orig = util.load_file(CA_CERT_CONFIG) - cur_cont = '\n'.join([line for line in orig.splitlines() - if line != CA_CERT_FILENAME]) - out = "%s\n%s\n" % (cur_cont.rstrip(), CA_CERT_FILENAME) + if os.stat(CA_CERT_CONFIG).st_size == 0: + # If the CA_CERT_CONFIG file is empty (i.e. all existing + # CA certs have been deleted) then simply output a single + # line with the cloud-init cert filename. + out = "%s\n" % CA_CERT_FILENAME + else: + # Append cert filename to CA_CERT_CONFIG file. + # We have to strip the content because blank lines in the file + # causes subsequent entries to be ignored. (LP: #1077020) + orig = util.load_file(CA_CERT_CONFIG) + cur_cont = '\n'.join([line for line in orig.splitlines() + if line != CA_CERT_FILENAME]) + out = "%s\n%s\n" % (cur_cont.rstrip(), CA_CERT_FILENAME) util.write_file(CA_CERT_CONFIG, out, omode="wb") diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index db0cdf9b..c1aff181 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -201,6 +201,28 @@ class TestAddCaCerts(TestCase): mock_load.assert_called_once_with("/etc/ca-certificates.conf") + def test_single_cert_to_empty_existing_ca_file(self): + """Test adding a single certificate to the trusted CAs + when existing ca-certificates.conf is empty""" + cert = "CERT1\nLINE2\nLINE3" + + expected = "cloud-init-ca-certs.crt\n" + + with ExitStack() as mocks: + mock_write = mocks.enter_context( + mock.patch.object(util, 'write_file', autospec=True)) + mock_stat = mocks.enter_context( + mock.patch("cloudinit.config.cc_ca_certs.os.stat") + ) + mock_stat.return_value.st_size = 0 + + cc_ca_certs.add_ca_certs([cert]) + + mock_write.assert_has_calls([ + mock.call("/usr/share/ca-certificates/cloud-init-ca-certs.crt", + cert, mode=0o644), + mock.call("/etc/ca-certificates.conf", expected, omode="wb")]) + def test_multiple_certs(self): """Test adding multiple certificates to the trusted CAs.""" certs = ["CERT1\nLINE2\nLINE3", "CERT2\nLINE2\nLINE3"] -- cgit v1.2.3 From 39251a6786b8d6b10903b194df6dfe859176fe5c Mon Sep 17 00:00:00 2001 From: Xiao Liang Date: Mon, 10 Aug 2020 07:36:38 -0700 Subject: Fix a typo in apt pipelining module (#525) * Fix a typo in apt pipelining module Changed `whcih` to `which`. * Update .github-cla-signers I have signed the CLA on Canonical's site, adding my username to list of CLA signers. * Update .github-cla-signers I need to sort the list alphabetically. --- cloudinit/config/cc_apt_pipelining.py | 2 +- tools/.github-cla-signers | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index 225d0905..aa186ce2 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -9,7 +9,7 @@ Apt Pipelining -------------- **Summary:** configure apt pipelining -This module configures apt's ``Acquite::http::Pipeline-Depth`` option, whcih +This module configures apt's ``Acquite::http::Pipeline-Depth`` option, which controls how apt handles HTTP pipelining. It may be useful for pipelining to be disabled, because some web servers, such as S3 do not pipeline properly (LP: #948461). The ``apt_pipelining`` config key may be set to ``false`` to disable diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 5f4dc79b..1b625fc3 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -5,6 +5,7 @@ candlerb dermotbradley dhensby eandersson +izzyleung landon912 lucasmoura marlluslustosa -- cgit v1.2.3 From 4fb7eab7989b8eb80b4451ea325311c5dc97e3bc Mon Sep 17 00:00:00 2001 From: sshedi <53473811+sshedi@users.noreply.github.com> Date: Thu, 13 Aug 2020 23:02:09 +0530 Subject: Fix 'Users and Groups' configuration documentation (#530) Few of the 'User and Groups' configurations in cloud-config have no effect on already existing users. This was not documented earlier. This change set adds that information to documentation. Signed-off-by: Shreenidhi Shedi --- cloudinit/config/cc_users_groups.py | 7 +++++++ doc/examples/cloud-config-user-groups.txt | 5 +++++ tools/.github-cla-signers | 1 + 3 files changed, 13 insertions(+) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index 13764e60..426498a3 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -78,6 +78,13 @@ config keys for an entry in ``users`` are as follows: If specifying a sudo rule for a user, ensure that the syntax for the rule is valid, as it is not checked by cloud-init. +.. note:: + Most of these configuration options will not be honored if the user + already exists. Following options are the exceptions and they are + applicable on already-existing users: + - 'plain_text_passwd', 'hashed_passwd', 'lock_passwd', 'sudo', + 'ssh_authorized_keys', 'ssh_redirect_user'. + **Internal name:** ``cc_users_groups`` **Module frequency:** per instance diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt index aa14a6dd..b593cdd1 100644 --- a/doc/examples/cloud-config-user-groups.txt +++ b/doc/examples/cloud-config-user-groups.txt @@ -7,6 +7,11 @@ groups: - cloud-users # Add users to the system. Users are added after groups are added. +# Note: Most of these configuration options will not be honored if the user +# already exists. Following options are the exceptions and they are +# applicable on already-existing users: +# - 'plain_text_passwd', 'hashed_passwd', 'lock_passwd', 'sudo', +# 'ssh_authorized_keys', 'ssh_redirect_user'. users: - default - name: foobar diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index fcaf752b..50473bf7 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -14,6 +14,7 @@ matthewruffell nishigori onitake smoser +sshedi TheRealFalcon tomponline tsanghan -- cgit v1.2.3 From b749548a9eb43b34cce64f8688107645411abc8c Mon Sep 17 00:00:00 2001 From: Eduardo Otubo Date: Tue, 18 Aug 2020 23:12:02 +0200 Subject: Detect kernel version before swap file creation (#428) According to man page `man 8 swapon', "Preallocated swap files are supported on XFS since Linux 4.18". This patch checks for kernel version before attepting to create swapfile, using dd for XFS only on kernel versions <= 4.18 or btrfs. Add new func util.kernel_version which returns a tuple of ints (major, minor) Signed-off-by: Eduardo Otubo otubo@redhat.com --- cloudinit/config/cc_mounts.py | 8 +- cloudinit/util.py | 4 + .../unittests/test_handler/test_handler_mounts.py | 107 +++++++++++++++++++++ tests/unittests/test_util.py | 16 +++ 4 files changed, 132 insertions(+), 3 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 773b8285..54f2f878 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -65,7 +65,7 @@ swap file is created. from string import whitespace import logging -import os.path +import os import re from cloudinit import type_utils @@ -263,7 +263,8 @@ def create_swapfile(fname: str, size: str) -> None: fstype = util.get_mount_info(swap_dir)[1] - if fstype in ("xfs", "btrfs"): + if (fstype == "xfs" and + util.kernel_version() < (4, 18)) or fstype == "btrfs": create_swap(fname, size, "dd") else: try: @@ -273,7 +274,8 @@ def create_swapfile(fname: str, size: str) -> None: LOG.warning("Will attempt with dd.") create_swap(fname, size, "dd") - util.chmod(fname, 0o600) + if os.path.exists(fname): + util.chmod(fname, 0o600) try: subp.subp(['mkswap', fname]) except subp.ProcessExecutionError: diff --git a/cloudinit/util.py b/cloudinit/util.py index 624c560d..edd37039 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -68,6 +68,10 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], ['lxc-is-container']) +def kernel_version(): + return tuple(map(int, os.uname().release.split('.')[:2])) + + @lru_cache() def get_dpkg_architecture(target=None): """Return the sanitized string output by `dpkg --print-architecture`. diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index e193f4d4..e87069f6 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -134,6 +134,113 @@ class TestSanitizeDevname(test_helpers.FilesystemMockingTestCase): cc_mounts.sanitize_devname(disk_path, None, mock.Mock())) +class TestSwapFileCreation(test_helpers.FilesystemMockingTestCase): + + def setUp(self): + super(TestSwapFileCreation, self).setUp() + self.new_root = self.tmp_dir() + self.patchOS(self.new_root) + + self.fstab_path = os.path.join(self.new_root, 'etc/fstab') + self.swap_path = os.path.join(self.new_root, 'swap.img') + self._makedirs('/etc') + + self.add_patch('cloudinit.config.cc_mounts.FSTAB_PATH', + 'mock_fstab_path', + self.fstab_path, + autospec=False) + + self.add_patch('cloudinit.config.cc_mounts.subp.subp', + 'm_subp_subp') + + self.add_patch('cloudinit.config.cc_mounts.util.mounts', + 'mock_util_mounts', + return_value={ + '/dev/sda1': {'fstype': 'ext4', + 'mountpoint': '/', + 'opts': 'rw,relatime,discard' + }}) + + self.mock_cloud = mock.Mock() + self.mock_log = mock.Mock() + self.mock_cloud.device_name_to_device = self.device_name_to_device + + self.cc = { + 'swap': { + 'filename': self.swap_path, + 'size': '512', + 'maxsize': '512'}} + + def _makedirs(self, directory): + directory = os.path.join(self.new_root, directory.lstrip('/')) + if not os.path.exists(directory): + os.makedirs(directory) + + def device_name_to_device(self, path): + if path == 'swap': + return self.swap_path + else: + dev = None + + return dev + + @mock.patch('cloudinit.util.get_mount_info') + @mock.patch('cloudinit.util.kernel_version') + def test_swap_creation_method_fallocate_on_xfs(self, m_kernel_version, + m_get_mount_info): + m_kernel_version.return_value = (4, 20) + m_get_mount_info.return_value = ["", "xfs"] + + cc_mounts.handle(None, self.cc, self.mock_cloud, self.mock_log, []) + self.m_subp_subp.assert_has_calls([ + mock.call(['fallocate', '-l', '0M', self.swap_path], capture=True), + mock.call(['mkswap', self.swap_path]), + mock.call(['swapon', '-a'])]) + + @mock.patch('cloudinit.util.get_mount_info') + @mock.patch('cloudinit.util.kernel_version') + def test_swap_creation_method_xfs(self, m_kernel_version, + m_get_mount_info): + m_kernel_version.return_value = (3, 18) + m_get_mount_info.return_value = ["", "xfs"] + + cc_mounts.handle(None, self.cc, self.mock_cloud, self.mock_log, []) + self.m_subp_subp.assert_has_calls([ + mock.call(['dd', 'if=/dev/zero', + 'of=' + self.swap_path, + 'bs=1M', 'count=0'], capture=True), + mock.call(['mkswap', self.swap_path]), + mock.call(['swapon', '-a'])]) + + @mock.patch('cloudinit.util.get_mount_info') + @mock.patch('cloudinit.util.kernel_version') + def test_swap_creation_method_btrfs(self, m_kernel_version, + m_get_mount_info): + m_kernel_version.return_value = (4, 20) + m_get_mount_info.return_value = ["", "btrfs"] + + cc_mounts.handle(None, self.cc, self.mock_cloud, self.mock_log, []) + self.m_subp_subp.assert_has_calls([ + mock.call(['dd', 'if=/dev/zero', + 'of=' + self.swap_path, + 'bs=1M', 'count=0'], capture=True), + mock.call(['mkswap', self.swap_path]), + mock.call(['swapon', '-a'])]) + + @mock.patch('cloudinit.util.get_mount_info') + @mock.patch('cloudinit.util.kernel_version') + def test_swap_creation_method_ext4(self, m_kernel_version, + m_get_mount_info): + m_kernel_version.return_value = (5, 14) + m_get_mount_info.return_value = ["", "ext4"] + + cc_mounts.handle(None, self.cc, self.mock_cloud, self.mock_log, []) + self.m_subp_subp.assert_has_calls([ + mock.call(['fallocate', '-l', '0M', self.swap_path], capture=True), + mock.call(['mkswap', self.swap_path]), + mock.call(['swapon', '-a'])]) + + class TestFstabHandling(test_helpers.FilesystemMockingTestCase): swap_path = '/dev/sdb1' diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index aa5352e9..fc557469 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -968,6 +968,22 @@ class TestGetProcEnv(helpers.TestCase): self.assertEqual(my_ppid, util.get_proc_ppid(my_pid)) +class TestKernelVersion(): + """test kernel version function""" + + params = [ + ('5.6.19-300.fc32.x86_64', (5, 6)), + ('4.15.0-101-generic', (4, 15)), + ('3.10.0-1062.12.1.vz7.131.10', (3, 10)), + ('4.18.0-144.el8.x86_64', (4, 18))] + + @mock.patch('os.uname') + @pytest.mark.parametrize("uname_release,expected", params) + def test_kernel_version(self, m_uname, uname_release, expected): + m_uname.return_value.release = uname_release + assert expected == util.kernel_version() + + class TestFindDevs: @mock.patch('cloudinit.subp.subp') def test_find_devs_with(self, m_subp): -- cgit v1.2.3 From 79a8ce7e714ae1686c10bff77612eab0f6eccc95 Mon Sep 17 00:00:00 2001 From: dermotbradley Date: Thu, 20 Aug 2020 00:18:25 +0100 Subject: Add Alpine Linux support. (#535) Add new module cc_apk_configure for creating Alpine /etc/apk/repositories file. Modify cc_ca_certs, cc_ntp, cc_power_state_change, and cc_resolv_conf for Alpine. Add Alpine template files for Chrony and Busybox NTP support. Add Alpine template file for /etc/hosts. --- README.md | 2 +- cloudinit/config/cc_apk_configure.py | 263 ++++++++++++++++++ cloudinit/config/cc_ca_certs.py | 22 +- cloudinit/config/cc_ntp.py | 61 ++++- cloudinit/config/cc_power_state_change.py | 57 +++- cloudinit/config/cc_resolv_conf.py | 4 +- cloudinit/distros/__init__.py | 7 +- cloudinit/distros/alpine.py | 165 ++++++++++++ cloudinit/util.py | 3 +- config/cloud.cfg.tmpl | 21 +- doc/rtd/topics/availability.rst | 13 +- doc/rtd/topics/instancedata.rst | 1 + doc/rtd/topics/modules.rst | 1 + doc/rtd/topics/network-config.rst | 2 +- templates/chrony.conf.alpine.tmpl | 38 +++ templates/hosts.alpine.tmpl | 28 ++ templates/ntp.conf.alpine.tmpl | 10 + tests/unittests/test_cli.py | 2 +- .../test_handler/test_handler_apk_configure.py | 299 +++++++++++++++++++++ .../test_handler/test_handler_ca_certs.py | 11 +- tests/unittests/test_handler/test_handler_ntp.py | 129 ++++++--- .../test_handler/test_handler_power_state.py | 29 +- tests/unittests/test_handler/test_schema.py | 1 + tools/render-cloudcfg | 5 +- 24 files changed, 1068 insertions(+), 106 deletions(-) create mode 100644 cloudinit/config/cc_apk_configure.py create mode 100644 cloudinit/distros/alpine.py create mode 100644 templates/chrony.conf.alpine.tmpl create mode 100644 templates/hosts.alpine.tmpl create mode 100644 templates/ntp.conf.alpine.tmpl create mode 100644 tests/unittests/test_handler/test_handler_apk_configure.py (limited to 'cloudinit/config') diff --git a/README.md b/README.md index a3455135..435405da 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ get in contact with that distribution and send them our way! | Supported OSes | Supported Public Clouds | Supported Private Clouds | | --- | --- | --- | -| Ubuntu
SLES/openSUSE
RHEL/CentOS
Fedora
Gentoo Linux
Debian
ArchLinux
FreeBSD
NetBSD
OpenBSD










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
Digital Ocean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)















| +| Alpine Linux
ArchLinux
Debian
Fedora
FreeBSD
Gentoo Linux
NetBSD
OpenBSD
RHEL/CentOS
SLES/openSUSE
Ubuntu










| Amazon Web Services
Microsoft Azure
Google Cloud Platform
Oracle Cloud Infrastructure
Softlayer
Rackspace Public Cloud
IBM Cloud
Digital Ocean
Bigstep
Hetzner
Joyent
CloudSigma
Alibaba Cloud
OVH
OpenNebula
Exoscale
Scaleway
CloudStack
AltCloud
SmartOS
HyperOne
Rootbox
| Bare metal installs
OpenStack
LXD
KVM
Metal-as-a-Service (MAAS)















| ## To start developing cloud-init diff --git a/cloudinit/config/cc_apk_configure.py b/cloudinit/config/cc_apk_configure.py new file mode 100644 index 00000000..84d7a0b6 --- /dev/null +++ b/cloudinit/config/cc_apk_configure.py @@ -0,0 +1,263 @@ +# Copyright (c) 2020 Dermot Bradley +# +# Author: Dermot Bradley +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""Apk Configure: Configures apk repositories file.""" + +from textwrap import dedent + +from cloudinit import log as logging +from cloudinit import temp_utils +from cloudinit import templater +from cloudinit import util +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + + +# If no mirror is specified then use this one +DEFAULT_MIRROR = "https://alpine.global.ssl.fastly.net/alpine" + + +REPOSITORIES_TEMPLATE = """\ +## template:jinja +# +# Created by cloud-init +# +# This file is written on first boot of an instance +# + +{{ alpine_baseurl }}/{{ alpine_version }}/main +{% if community_enabled -%} +{{ alpine_baseurl }}/{{ alpine_version }}/community +{% endif -%} +{% if testing_enabled -%} +{% if alpine_version != 'edge' %} +# +# Testing - using with non-Edge installation may cause problems! +# +{% endif %} +{{ alpine_baseurl }}/edge/testing +{% endif %} +{% if local_repo != '' %} + +# +# Local repo +# +{{ local_repo }}/{{ alpine_version }} +{% endif %} + +""" + + +frequency = PER_INSTANCE +distros = ['alpine'] +schema = { + 'id': 'cc_apk_configure', + 'name': 'APK Configure', + 'title': 'Configure apk repositories file', + 'description': dedent("""\ + This module handles configuration of the /etc/apk/repositories file. + + .. note:: + To ensure that apk configuration is valid yaml, any strings + containing special characters, especially ``:`` should be quoted. + """), + 'distros': distros, + 'examples': [ + dedent("""\ + # Keep the existing /etc/apk/repositories file unaltered. + apk_repos: + preserve_repositories: true + """), + dedent("""\ + # Create repositories file for Alpine v3.12 main and community + # using default mirror site. + apk_repos: + alpine_repo: + community_enabled: true + version: 'v3.12' + """), + dedent("""\ + # Create repositories file for Alpine Edge main, community, and + # testing using a specified mirror site and also a local repo. + apk_repos: + alpine_repo: + base_url: 'https://some-alpine-mirror/alpine' + community_enabled: true + testing_enabled: true + version: 'edge' + local_repo_base_url: 'https://my-local-server/local-alpine' + """), + ], + 'frequency': frequency, + 'type': 'object', + 'properties': { + 'apk_repos': { + 'type': 'object', + 'properties': { + 'preserve_repositories': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + By default, cloud-init will generate a new repositories + file ``/etc/apk/repositories`` based on any valid + configuration settings specified within a apk_repos + section of cloud config. To disable this behavior and + preserve the repositories file from the pristine image, + set ``preserve_repositories`` to ``true``. + + The ``preserve_repositories`` option overrides + all other config keys that would alter + ``/etc/apk/repositories``. + """) + }, + 'alpine_repo': { + 'type': ['object', 'null'], + 'properties': { + 'base_url': { + 'type': 'string', + 'default': DEFAULT_MIRROR, + 'description': dedent("""\ + The base URL of an Alpine repository, or + mirror, to download official packages from. + If not specified then it defaults to ``{}`` + """.format(DEFAULT_MIRROR)) + }, + 'community_enabled': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + Whether to add the Community repo to the + repositories file. By default the Community + repo is not included. + """) + }, + 'testing_enabled': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + Whether to add the Testing repo to the + repositories file. By default the Testing + repo is not included. It is only recommended + to use the Testing repo on a machine running + the ``Edge`` version of Alpine as packages + installed from Testing may have dependancies + that conflict with those in non-Edge Main or + Community repos." + """) + }, + 'version': { + 'type': 'string', + 'description': dedent("""\ + The Alpine version to use (e.g. ``v3.12`` or + ``edge``) + """) + }, + }, + 'required': ['version'], + 'minProperties': 1, + 'additionalProperties': False, + }, + 'local_repo_base_url': { + 'type': 'string', + 'description': dedent("""\ + The base URL of an Alpine repository containing + unofficial packages + """) + } + }, + 'required': [], + 'minProperties': 1, # Either preserve_repositories or alpine_repo + 'additionalProperties': False, + } + } +} + +__doc__ = get_schema_doc(schema) + + +def handle(name, cfg, cloud, log, _args): + """ + Call to handle apk_repos sections in cloud-config file. + + @param name: The module name "apk-configure" from cloud.cfg + @param cfg: A nested dict containing the entire cloud config contents. + @param cloud: The CloudInit object in use. + @param log: Pre-initialized Python logger object to use for logging. + @param _args: Any module arguments from cloud.cfg + """ + + # If there is no "apk_repos" section in the configuration + # then do nothing. + apk_section = cfg.get('apk_repos') + if not apk_section: + LOG.debug(("Skipping module named %s," + " no 'apk_repos' section found"), name) + return + + validate_cloudconfig_schema(cfg, schema) + + # If "preserve_repositories" is explicitly set to True in + # the configuration do nothing. + if util.get_cfg_option_bool(apk_section, 'preserve_repositories', False): + LOG.debug(("Skipping module named %s," + " 'preserve_repositories' is set"), name) + return + + # If there is no "alpine_repo" subsection of "apk_repos" present in the + # configuration then do nothing, as at least "version" is required to + # create valid repositories entries. + alpine_repo = apk_section.get('alpine_repo') + if not alpine_repo: + LOG.debug(("Skipping module named %s," + " no 'alpine_repo' configuration found"), name) + return + + # If there is no "version" value present in configuration then do nothing. + alpine_version = alpine_repo.get('version') + if not alpine_version: + LOG.debug(("Skipping module named %s," + " 'version' not specified in alpine_repo"), name) + return + + local_repo = apk_section.get('local_repo_base_url', '') + + _write_repositories_file(alpine_repo, alpine_version, local_repo) + + +def _write_repositories_file(alpine_repo, alpine_version, local_repo): + """ + Write the /etc/apk/repositories file with the specified entries. + + @param alpine_repo: A nested dict of the alpine_repo configuration. + @param alpine_version: A string of the Alpine version to use. + @param local_repo: A string containing the base URL of a local repo. + """ + + repo_file = '/etc/apk/repositories' + + alpine_baseurl = alpine_repo.get('base_url', DEFAULT_MIRROR) + + params = {'alpine_baseurl': alpine_baseurl, + 'alpine_version': alpine_version, + 'community_enabled': alpine_repo.get('community_enabled'), + 'testing_enabled': alpine_repo.get('testing_enabled'), + 'local_repo': local_repo} + + tfile = temp_utils.mkstemp(prefix='template_name-', suffix=".tmpl") + template_fn = tfile[1] # Filepath is second item in tuple + util.write_file(template_fn, content=REPOSITORIES_TEMPLATE) + + LOG.debug('Generating Alpine repository configuration file: %s', + repo_file) + templater.render_to_file(template_fn, repo_file, params) + # Clean up temporary template + util.del_file(template_fn) + + +# vi: ts=4 expandtab diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 910b78de..3c453d91 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -16,11 +16,16 @@ can be removed from the system with the configuration option certificates must be specified using valid yaml. in order to specify a multiline certificate, the yaml multiline list syntax must be used +.. note:: + For Alpine Linux the "remove-defaults" functionality works if the + ca-certificates package is installed but not if the + ca-certificates-bundle package is installed. + **Internal name:** ``cc_ca_certs`` **Module frequency:** per instance -**Supported distros:** ubuntu, debian +**Supported distros:** alpine, debian, ubuntu **Config keys**:: @@ -45,7 +50,7 @@ CA_CERT_CONFIG = "/etc/ca-certificates.conf" CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" CA_CERT_FULL_PATH = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) -distros = ['ubuntu', 'debian'] +distros = ['alpine', 'debian', 'ubuntu'] def update_ca_certs(): @@ -83,7 +88,7 @@ def add_ca_certs(certs): util.write_file(CA_CERT_CONFIG, out, omode="wb") -def remove_default_ca_certs(): +def remove_default_ca_certs(distro_name): """ Removes all default trusted CA certificates from the system. To actually apply the change you must also call L{update_ca_certs}. @@ -91,11 +96,14 @@ def remove_default_ca_certs(): util.delete_dir_contents(CA_CERT_PATH) util.delete_dir_contents(CA_CERT_SYSTEM_PATH) util.write_file(CA_CERT_CONFIG, "", mode=0o644) - debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" - subp.subp(('debconf-set-selections', '-'), debconf_sel) + + if distro_name != 'alpine': + debconf_sel = ( + "ca-certificates ca-certificates/trust_new_crts " + "select no") + subp.subp(('debconf-set-selections', '-'), debconf_sel) -def handle(name, cfg, _cloud, log, _args): +def handle(name, cfg, cloud, log, _args): """ Call to handle ca-cert sections in cloud-config file. @@ -117,7 +125,7 @@ def handle(name, cfg, _cloud, log, _args): # default trusted CA certs first. if ca_cert_cfg.get("remove-defaults", False): log.debug("Removing default certificates") - remove_default_ca_certs() + remove_default_ca_certs(cloud.distro.name) # If we are given any new trusted CA certs to add, add them. if "trusted" in ca_cert_cfg: diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 7d3f73ff..3d7279d6 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -24,7 +24,8 @@ LOG = logging.getLogger(__name__) frequency = PER_INSTANCE NTP_CONF = '/etc/ntp.conf' NR_POOL_SERVERS = 4 -distros = ['centos', 'debian', 'fedora', 'opensuse', 'rhel', 'sles', 'ubuntu'] +distros = ['alpine', 'centos', 'debian', 'fedora', 'opensuse', 'rhel', + 'sles', 'ubuntu'] NTP_CLIENT_CONFIG = { 'chrony': { @@ -63,6 +64,17 @@ NTP_CLIENT_CONFIG = { # This is Distro-specific configuration overrides of the base config DISTRO_CLIENT_CONFIG = { + 'alpine': { + 'chrony': { + 'confpath': '/etc/chrony/chrony.conf', + 'service_name': 'chronyd', + }, + 'ntp': { + 'confpath': '/etc/ntp.conf', + 'packages': [], + 'service_name': 'ntpd', + }, + }, 'debian': { 'chrony': { 'confpath': '/etc/chrony/chrony.conf', @@ -114,11 +126,11 @@ schema = { Handle ntp configuration. If ntp is not installed on the system and ntp configuration is specified, ntp will be installed. If there is a default ntp config file in the image or one is present in the - distro's ntp package, it will be copied to ``/etc/ntp.conf.dist`` - before any changes are made. A list of ntp pools and ntp servers can - be provided under the ``ntp`` config key. If no ntp ``servers`` or - ``pools`` are provided, 4 pools will be used in the format - ``{0-3}.{distro}.pool.ntp.org``."""), + distro's ntp package, it will be copied to a file with ``.dist`` + appended to the filename before any changes are made. A list of ntp + pools and ntp servers can be provided under the ``ntp`` config key. + If no ntp ``servers`` or ``pools`` are provided, 4 pools will be used + in the format ``{0-3}.{distro}.pool.ntp.org``."""), 'distros': distros, 'examples': [ dedent("""\ @@ -171,7 +183,10 @@ schema = { 'description': dedent("""\ List of ntp pools. If both pools and servers are empty, 4 default pool servers will be provided of - the format ``{0-3}.{distro}.pool.ntp.org``.""") + the format ``{0-3}.{distro}.pool.ntp.org``. NOTE: + for Alpine Linux when using the Busybox NTP client + this setting will be ignored due to the limited + functionality of Busybox's ntpd.""") }, 'servers': { 'type': 'array', @@ -364,21 +379,30 @@ def generate_server_names(distro): """ names = [] pool_distro = distro - # For legal reasons x.pool.sles.ntp.org does not exist, - # use the opensuse pool + if distro == 'sles': + # For legal reasons x.pool.sles.ntp.org does not exist, + # use the opensuse pool pool_distro = 'opensuse' + elif distro == 'alpine': + # Alpine-specific pool (i.e. x.alpine.pool.ntp.org) does not exist + # so use general x.pool.ntp.org instead. + pool_distro = '' + for x in range(0, NR_POOL_SERVERS): - name = "%d.%s.pool.ntp.org" % (x, pool_distro) - names.append(name) + names.append(".".join( + [n for n in [str(x)] + [pool_distro] + ['pool.ntp.org'] if n])) + return names -def write_ntp_config_template(distro_name, servers=None, pools=None, - path=None, template_fn=None, template=None): +def write_ntp_config_template(distro_name, service_name=None, servers=None, + pools=None, path=None, template_fn=None, + template=None): """Render a ntp client configuration for the specified client. @param distro_name: string. The distro class name. + @param service_name: string. The name of the NTP client service. @param servers: A list of strings specifying ntp servers. Defaults to empty list. @param pools: A list of strings specifying ntp pools. Defaults to empty @@ -397,7 +421,14 @@ def write_ntp_config_template(distro_name, servers=None, pools=None, if not pools: pools = [] - if len(servers) == 0 and len(pools) == 0: + if (len(servers) == 0 and distro_name == 'alpine' and + service_name == 'ntpd'): + # Alpine's Busybox ntpd only understands "servers" configuration + # and not "pool" configuration. + servers = generate_server_names(distro_name) + LOG.debug( + 'Adding distro default ntp servers: %s', ','.join(servers)) + elif len(servers) == 0 and len(pools) == 0: pools = generate_server_names(distro_name) LOG.debug( 'Adding distro default ntp pool servers: %s', ','.join(pools)) @@ -532,6 +563,8 @@ def handle(name, cfg, cloud, log, _args): raise RuntimeError(msg) write_ntp_config_template(cloud.distro.name, + service_name=ntp_client_config.get( + 'service_name'), servers=ntp_cfg.get('servers', []), pools=ntp_cfg.get('pools', []), path=ntp_client_config.get('confpath'), diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 41ffb46c..ab953a0d 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -22,9 +22,8 @@ The ``delay`` key specifies a duration to be added onto any shutdown command used. Therefore, if a 5 minute delay and a 120 second shutdown are specified, the maximum amount of time between cloud-init starting and the system shutting down is 7 minutes, and the minimum amount of time is 5 minutes. The ``delay`` -key must have an argument in a form that the ``shutdown`` utility recognizes. -The most common format is the form ``+5`` for 5 minutes. See ``man shutdown`` -for more options. +key must have an argument in either the form ``+5`` for 5 minutes or ``now`` +for immediate shutdown. Optionally, a command can be run to determine whether or not the system should shut down. The command to be run should be specified in the @@ -33,6 +32,10 @@ the system should shut down. The command to be run should be specified in the ``condition`` key is omitted or the command specified by the ``condition`` key returns 0. +.. note:: + With Alpine Linux any message value specified is ignored as Alpine's halt, + poweroff, and reboot commands do not support broadcasting a message. + **Internal name:** ``cc_power_state_change`` **Module frequency:** per instance @@ -112,9 +115,9 @@ def check_condition(cond, log=None): return False -def handle(_name, cfg, _cloud, log, _args): +def handle(_name, cfg, cloud, log, _args): try: - (args, timeout, condition) = load_power_state(cfg) + (args, timeout, condition) = load_power_state(cfg, cloud.distro.name) if args is None: log.debug("no power_state provided. doing nothing") return @@ -141,7 +144,19 @@ def handle(_name, cfg, _cloud, log, _args): condition, execmd, [args, devnull_fp]) -def load_power_state(cfg): +def convert_delay(delay, fmt=None, scale=None): + if not fmt: + fmt = "+%s" + if not scale: + scale = 1 + + if delay != "now": + delay = fmt % int(int(delay) * int(scale)) + + return delay + + +def load_power_state(cfg, distro_name): # returns a tuple of shutdown_command, timeout # shutdown_command is None if no config found pstate = cfg.get('power_state') @@ -161,20 +176,34 @@ def load_power_state(cfg): (','.join(opt_map.keys()), mode)) delay = pstate.get("delay", "now") - # convert integer 30 or string '30' to '+30' + message = pstate.get("message") + scale = 1 + fmt = "+%s" + command = ["shutdown", opt_map[mode]] + + if distro_name == 'alpine': + # Convert integer 30 or string '30' to '1800' (seconds) as Alpine's + # halt/poweroff/reboot commands take seconds rather than minutes. + scale = 60 + # No "+" in front of delay value as not supported by Alpine's commands. + fmt = "%s" + if delay == "now": + # Alpine's commands do not understand "now". + delay = "0" + command = [mode, "-d"] + # Alpine's commands don't support a message. + message = None + try: - delay = "+%s" % int(delay) + delay = convert_delay(delay, fmt=fmt, scale=scale) except ValueError: - pass - - if delay != "now" and not re.match(r"\+[0-9]+", delay): raise TypeError( "power_state[delay] must be 'now' or '+m' (minutes)." " found '%s'." % delay) - args = ["shutdown", opt_map[mode], delay] - if pstate.get("message"): - args.append(pstate.get("message")) + args = command + [delay] + if message: + args.append(message) try: timeout = float(pstate.get('timeout', 30.0)) diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index 69f4768a..519e66eb 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -30,7 +30,7 @@ are configured correctly. **Module frequency:** per instance -**Supported distros:** fedora, rhel, sles +**Supported distros:** alpine, fedora, rhel, sles **Config keys**:: @@ -55,7 +55,7 @@ LOG = logging.getLogger(__name__) frequency = PER_INSTANCE -distros = ['fedora', 'opensuse', 'rhel', 'sles'] +distros = ['alpine', 'fedora', 'opensuse', 'rhel', 'sles'] def generate_resolv_conf(template_fn, params, target_fname="/etc/resolv.conf"): diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index c7163e1c..effb4276 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -40,12 +40,13 @@ from .networking import LinuxNetworking ALL_DISTROS = 'all' OSFAMILIES = { + 'alpine': ['alpine'], + 'arch': ['arch'], 'debian': ['debian', 'ubuntu'], - 'redhat': ['amazon', 'centos', 'fedora', 'rhel'], - 'gentoo': ['gentoo'], 'freebsd': ['freebsd'], + 'gentoo': ['gentoo'], + 'redhat': ['amazon', 'centos', 'fedora', 'rhel'], 'suse': ['opensuse', 'sles'], - 'arch': ['arch'], } LOG = logging.getLogger(__name__) diff --git a/cloudinit/distros/alpine.py b/cloudinit/distros/alpine.py new file mode 100644 index 00000000..e42443fc --- /dev/null +++ b/cloudinit/distros/alpine.py @@ -0,0 +1,165 @@ +# Copyright (C) 2016 Matt Dainty +# Copyright (C) 2020 Dermot Bradley +# +# Author: Matt Dainty +# Author: Dermot Bradley +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import distros +from cloudinit import helpers +from cloudinit import log as logging +from cloudinit import subp +from cloudinit import util + +from cloudinit.distros.parsers.hostname import HostnameConf + +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + +NETWORK_FILE_HEADER = """\ +# This file is generated from information provided by the datasource. Changes +# to it will not persist across an instance reboot. To disable cloud-init's +# network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} + +""" + + +class Distro(distros.Distro): + init_cmd = ['rc-service'] # init scripts + locale_conf_fn = "/etc/profile.d/locale.sh" + network_conf_fn = "/etc/network/interfaces" + renderer_configs = { + "eni": {"eni_path": network_conf_fn, + "eni_header": NETWORK_FILE_HEADER} + } + + def __init__(self, name, cfg, paths): + distros.Distro.__init__(self, name, cfg, paths) + # This will be used to restrict certain + # calls from repeatly happening (when they + # should only happen say once per instance...) + self._runner = helpers.Runners(paths) + self.default_locale = 'C.UTF-8' + self.osfamily = 'alpine' + cfg['ssh_svcname'] = 'sshd' + + def get_locale(self): + """The default locale for Alpine Linux is different than + cloud-init's DataSource default. + """ + return self.default_locale + + def apply_locale(self, locale, out_fn=None): + # Alpine has limited locale support due to musl library limitations + + if not locale: + locale = self.default_locale + if not out_fn: + out_fn = self.locale_conf_fn + + lines = [ + "#", + "# This file is created by cloud-init once per new instance boot", + "#", + "export CHARSET=UTF-8", + "export LANG=%s" % locale, + "export LC_COLLATE=C", + "", + ] + util.write_file(out_fn, "\n".join(lines), 0o644) + + def install_packages(self, pkglist): + self.update_package_sources() + self.package_command('add', pkgs=pkglist) + + def _write_network_config(self, netconfig): + return self._supported_write_network_config(netconfig) + + def _bring_up_interfaces(self, device_names): + use_all = False + for d in device_names: + if d == 'all': + use_all = True + if use_all: + return distros.Distro._bring_up_interface(self, '-a') + else: + return distros.Distro._bring_up_interfaces(self, device_names) + + def _write_hostname(self, your_hostname, out_fn): + conf = None + try: + # Try to update the previous one + # so lets see if we can read it first. + conf = self._read_hostname_conf(out_fn) + except IOError: + pass + if not conf: + conf = HostnameConf('') + conf.set_hostname(your_hostname) + util.write_file(out_fn, str(conf), 0o644) + + def _read_system_hostname(self): + sys_hostname = self._read_hostname(self.hostname_conf_fn) + return (self.hostname_conf_fn, sys_hostname) + + def _read_hostname_conf(self, filename): + conf = HostnameConf(util.load_file(filename)) + conf.parse() + return conf + + def _read_hostname(self, filename, default=None): + hostname = None + try: + conf = self._read_hostname_conf(filename) + hostname = conf.hostname + except IOError: + pass + if not hostname: + return default + return hostname + + def _get_localhost_ip(self): + return "127.0.1.1" + + def set_timezone(self, tz): + distros.set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) + + def package_command(self, command, args=None, pkgs=None): + if pkgs is None: + pkgs = [] + + cmd = ['apk'] + # Redirect output + cmd.append("--quiet") + + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): + cmd.extend(args) + + if command: + cmd.append(command) + + pkglist = util.expand_package_list('%s-%s', pkgs) + cmd.extend(pkglist) + + # Allow the output of this to flow outwards (ie not be captured) + subp.subp(cmd, capture=False) + + def update_package_sources(self): + self._runner.run("update-sources", self.package_command, + ["update"], freq=PER_INSTANCE) + + @property + def preferred_ntp_clients(self): + """Allow distro to determine the preferred ntp client list""" + if not self._preferred_ntp_clients: + self._preferred_ntp_clients = ['chrony', 'ntp'] + + return self._preferred_ntp_clients + +# vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index edd37039..dd263803 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -548,7 +548,8 @@ def system_info(): if system == "linux": linux_dist = info['dist'][0].lower() if linux_dist in ( - 'arch', 'centos', 'debian', 'fedora', 'rhel', 'suse'): + 'alpine', 'arch', 'centos', 'debian', 'fedora', 'rhel', + 'suse'): var = linux_dist elif linux_dist in ('ubuntu', 'linuxmint', 'mint'): var = 'ubuntu' diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index b44cbce7..2beb9b0c 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -21,7 +21,7 @@ disable_root: false disable_root: true {% endif %} -{% if variant in ["amazon", "centos", "fedora", "rhel"] %} +{% if variant in ["alpine", "amazon", "centos", "fedora", "rhel"] %} mount_default_fields: [~, ~, 'auto', 'defaults,nofail', '0', '2'] {% if variant == "amazon" %} resize_rootfs: noblock @@ -71,6 +71,9 @@ cloud_init_modules: - set_hostname - update_hostname - update_etc_hosts +{% if variant in ["alpine"] %} + - resolv_conf +{% endif %} {% if not variant.endswith("bsd") %} - ca-certs - rsyslog @@ -104,6 +107,9 @@ cloud_config_modules: {% if variant in ["suse"] %} - zypper-add-repo {% endif %} +{% if variant in ["alpine"] %} + - apk-configure +{% endif %} {% if variant not in ["freebsd", "netbsd"] %} - ntp {% endif %} @@ -145,7 +151,9 @@ cloud_final_modules: # (not accessible to handlers/transforms) system_info: # This will affect which distro class gets used -{% if variant in ["amazon", "arch", "centos", "debian", "fedora", "freebsd", "netbsd", "openbsd", "rhel", "suse", "ubuntu"] %} +{% if variant in ["alpine", "amazon", "arch", "centos", "debian", + "fedora", "freebsd", "netbsd", "openbsd", "rhel", + "suse", "ubuntu"] %} distro: {{ variant }} {% else %} # Unknown/fallback distro. @@ -196,7 +204,8 @@ system_info: primary: http://ports.ubuntu.com/ubuntu-ports security: http://ports.ubuntu.com/ubuntu-ports ssh_svcname: ssh -{% elif variant in ["amazon", "arch", "centos", "fedora", "rhel", "suse"] %} +{% elif variant in ["alpine", "amazon", "arch", "centos", "fedora", + "rhel", "suse"] %} # Default user name + that default users groups (if added/used) default_user: {% if variant == "amazon" %} @@ -210,13 +219,19 @@ system_info: {% endif %} {% if variant == "suse" %} groups: [cdrom, users] +{% elif variant == "alpine" %} + groups: [adm, sudo] {% elif variant == "arch" %} groups: [wheel, users] {% else %} groups: [wheel, adm, systemd-journal] {% endif %} sudo: ["ALL=(ALL) NOPASSWD:ALL"] +{% if variant == "alpine" %} + shell: /bin/ash +{% else %} shell: /bin/bash +{% endif %} # Other config here will be given to the distro class and/or path classes paths: cloud_dir: /var/lib/cloud/ diff --git a/doc/rtd/topics/availability.rst b/doc/rtd/topics/availability.rst index 84490460..8f56a7d2 100644 --- a/doc/rtd/topics/availability.rst +++ b/doc/rtd/topics/availability.rst @@ -17,16 +17,17 @@ Distributions Cloud-init has support across all major Linux distributions, FreeBSD, NetBSD and OpenBSD: -- Ubuntu -- SLES/openSUSE -- RHEL/CentOS -- Fedora -- Gentoo Linux -- Debian +- Alpine Linux - ArchLinux +- Debian +- Fedora - FreeBSD +- Gentoo Linux - NetBSD - OpenBSD +- RHEL/CentOS +- SLES/openSUSE +- Ubuntu Clouds ====== diff --git a/doc/rtd/topics/instancedata.rst b/doc/rtd/topics/instancedata.rst index 845098bb..255245a4 100644 --- a/doc/rtd/topics/instancedata.rst +++ b/doc/rtd/topics/instancedata.rst @@ -132,6 +132,7 @@ This shall be the distro name, version and release as determined by Example output: +- alpine, 3.12.0, '' - centos, 7.5, core - debian, 9, stretch - freebsd, 12.0-release-p10, diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index 9c9be804..e30fe0fe 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -6,6 +6,7 @@ Modules ******* .. contents:: Table of Contents +.. automodule:: cloudinit.config.cc_apk_configure .. automodule:: cloudinit.config.cc_apt_configure .. automodule:: cloudinit.config.cc_apt_pipelining .. automodule:: cloudinit.config.cc_bootcmd diff --git a/doc/rtd/topics/network-config.rst b/doc/rtd/topics/network-config.rst index 8eeadebf..08db04d8 100644 --- a/doc/rtd/topics/network-config.rst +++ b/doc/rtd/topics/network-config.rst @@ -165,7 +165,7 @@ supported formats. The following ``renderers`` are supported in cloud-init: - **ENI** /etc/network/interfaces or ``ENI`` is supported by the ``ifupdown`` package -found in Ubuntu and Debian. +found in Alpine Linux, Debian and Ubuntu. - **Netplan** diff --git a/templates/chrony.conf.alpine.tmpl b/templates/chrony.conf.alpine.tmpl new file mode 100644 index 00000000..45efc18c --- /dev/null +++ b/templates/chrony.conf.alpine.tmpl @@ -0,0 +1,38 @@ +## template:jinja +# Welcome to the chrony configuration file. See chrony.conf(5) for more +# information about usable directives. +{% if pools %}# pools +{% endif %} +{% for pool in pools -%} +pool {{pool}} iburst +{% endfor %} +{%- if servers %}# servers +{% endif %} +{% for server in servers -%} +server {{server}} iburst +{% endfor %} + +# This directive specifies the location of the file containing ID/key pairs for +# NTP authentication. +keyfile /etc/chrony/chrony.keys + +# This directive specifies the file into which chronyd will store the rate +# information. +driftfile /var/lib/chrony/chrony.drift + +# Uncomment the following line to turn logging on. +#log tracking measurements statistics + +# Log files location. +logdir /var/log/chrony + +# Stop bad estimates upsetting machine clock. +maxupdateskew 100.0 + +# This directive enables kernel synchronisation (every 11 minutes) of the +# real-time clock. Note that it can’t be used along with the 'rtcfile' directive. +rtcsync + +# Step the system clock instead of slewing it if the adjustment is larger than +# one second, but only in the first three clock updates. +makestep 1 3 diff --git a/templates/hosts.alpine.tmpl b/templates/hosts.alpine.tmpl new file mode 100644 index 00000000..33c1a941 --- /dev/null +++ b/templates/hosts.alpine.tmpl @@ -0,0 +1,28 @@ +## template:jinja +{# +This file /etc/cloud/templates/hosts.alpine.tmpl is only utilized +if enabled in cloud-config. Specifically, in order to enable it +you need to add the following to config: + manage_etc_hosts: True +-#} +# Your system has configured 'manage_etc_hosts' as True. +# As a result, if you wish for changes to this file to persist +# then you will need to either +# a.) make changes to the master file in /etc/cloud/templates/hosts.alpine.tmpl +# b.) change or remove the value of 'manage_etc_hosts' in +# /etc/cloud/cloud.cfg or cloud-config from user-data +# +# The following lines are desirable for IPv4 capable hosts +127.0.1.1 {{fqdn}} {{hostname}} +127.0.0.1 localhost.localdomain localhost +127.0.0.1 localhost4.localdomain4 localhost4 + +# The following lines are desirable for IPv6 capable hosts +::1 {{fqdn}} {{hostname}} +::1 localhost6.localdomain6 localhost6 + +fe00::0 ip6-localnet +ff00::0 ip6-mcastprefix +ff02::1 ip6-allnodes +ff02::2 ip6-allrouters +ff02::3 ip6-allhosts diff --git a/templates/ntp.conf.alpine.tmpl b/templates/ntp.conf.alpine.tmpl new file mode 100644 index 00000000..59ca8fc1 --- /dev/null +++ b/templates/ntp.conf.alpine.tmpl @@ -0,0 +1,10 @@ +## template:jinja +# /etc/ntp.conf +# +# Configuration for Busybox ntpd - it only supports "server" lines. + +{% if servers %}# Servers +{% endif %} +{% for server in servers -%} +server {{server}} +{% endfor %} diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 43d996b9..dcf0fe5a 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -224,7 +224,7 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): self._call_main(['cloud-init', 'devel', 'schema', '--docs', 'all']) expected_doc_sections = [ '**Supported distros:** all', - '**Supported distros:** centos, debian, fedora', + '**Supported distros:** alpine, centos, debian, fedora', '**Config schema**:\n **resize_rootfs:** (true/false/noblock)', '**Examples**::\n\n runcmd:\n - [ ls, -l, / ]\n' ] diff --git a/tests/unittests/test_handler/test_handler_apk_configure.py b/tests/unittests/test_handler/test_handler_apk_configure.py new file mode 100644 index 00000000..8acc0b33 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apk_configure.py @@ -0,0 +1,299 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +""" test_apk_configure +Test creation of repositories file +""" + +import logging +import os +import textwrap + +from cloudinit import (cloud, helpers, util) + +from cloudinit.config import cc_apk_configure +from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock) + +REPO_FILE = "/etc/apk/repositories" +DEFAULT_MIRROR_URL = "https://alpine.global.ssl.fastly.net/alpine" +CC_APK = 'cloudinit.config.cc_apk_configure' + + +class TestNoConfig(FilesystemMockingTestCase): + def setUp(self): + super(TestNoConfig, self).setUp() + self.add_patch(CC_APK + '._write_repositories_file', 'm_write_repos') + self.name = "apk-configure" + self.cloud_init = None + self.log = logging.getLogger("TestNoConfig") + self.args = [] + + def test_no_config(self): + """ + Test that nothing is done if no apk-configure + configuration is provided. + """ + config = util.get_builtin_cfg() + + cc_apk_configure.handle(self.name, config, self.cloud_init, + self.log, self.args) + + self.assertEqual(0, self.m_write_repos.call_count) + + +class TestConfig(FilesystemMockingTestCase): + def setUp(self): + super(TestConfig, self).setUp() + self.new_root = self.tmp_dir() + self.new_root = self.reRoot(root=self.new_root) + for dirname in ['tmp', 'etc/apk']: + util.ensure_dir(os.path.join(self.new_root, dirname)) + self.paths = helpers.Paths({'templates_dir': self.new_root}) + self.name = "apk-configure" + self.cloud = cloud.Cloud(None, self.paths, None, None, None) + self.log = logging.getLogger("TestNoConfig") + self.args = [] + + @mock.patch(CC_APK + '._write_repositories_file') + def test_no_repo_settings(self, m_write_repos): + """ + Test that nothing is written if the 'alpine-repo' key + is not present. + """ + config = {"apk_repos": {}} + + cc_apk_configure.handle(self.name, config, self.cloud, self.log, + self.args) + + self.assertEqual(0, m_write_repos.call_count) + + @mock.patch(CC_APK + '._write_repositories_file') + def test_empty_repo_settings(self, m_write_repos): + """ + Test that nothing is written if 'alpine_repo' list is empty. + """ + config = {"apk_repos": {"alpine_repo": []}} + + cc_apk_configure.handle(self.name, config, self.cloud, self.log, + self.args) + + self.assertEqual(0, m_write_repos.call_count) + + def test_only_main_repo(self): + """ + Test when only details of main repo is written to file. + """ + alpine_version = 'v3.12' + config = { + "apk_repos": { + "alpine_repo": { + "version": alpine_version + } + } + } + + cc_apk_configure.handle(self.name, config, self.cloud, self.log, + self.args) + + expected_content = textwrap.dedent("""\ + # + # Created by cloud-init + # + # This file is written on first boot of an instance + # + + {0}/{1}/main + + """.format(DEFAULT_MIRROR_URL, alpine_version)) + + self.assertEqual(expected_content, util.load_file(REPO_FILE)) + + def test_main_and_community_repos(self): + """ + Test when only details of main and community repos are + written to file. + """ + alpine_version = 'edge' + config = { + "apk_repos": { + "alpine_repo": { + "version": alpine_version, + "community_enabled": True + } + } + } + + cc_apk_configure.handle(self.name, config, self.cloud, self.log, + self.args) + + expected_content = textwrap.dedent("""\ + # + # Created by cloud-init + # + # This file is written on first boot of an instance + # + + {0}/{1}/main + {0}/{1}/community + + """.format(DEFAULT_MIRROR_URL, alpine_version)) + + self.assertEqual(expected_content, util.load_file(REPO_FILE)) + + def test_main_community_testing_repos(self): + """ + Test when details of main, community and testing repos + are written to file. + """ + alpine_version = 'v3.12' + config = { + "apk_repos": { + "alpine_repo": { + "version": alpine_version, + "community_enabled": True, + "testing_enabled": True + } + } + } + + cc_apk_configure.handle(self.name, config, self.cloud, self.log, + self.args) + + expected_content = textwrap.dedent("""\ + # + # Created by cloud-init + # + # This file is written on first boot of an instance + # + + {0}/{1}/main + {0}/{1}/community + # + # Testing - using with non-Edge installation may cause problems! + # + {0}/edge/testing + + """.format(DEFAULT_MIRROR_URL, alpine_version)) + + self.assertEqual(expected_content, util.load_file(REPO_FILE)) + + def test_edge_main_community_testing_repos(self): + """ + Test when details of main, community and testing repos + for Edge version of Alpine are written to file. + """ + alpine_version = 'edge' + config = { + "apk_repos": { + "alpine_repo": { + "version": alpine_version, + "community_enabled": True, + "testing_enabled": True + } + } + } + + cc_apk_configure.handle(self.name, config, self.cloud, self.log, + self.args) + + expected_content = textwrap.dedent("""\ + # + # Created by cloud-init + # + # This file is written on first boot of an instance + # + + {0}/{1}/main + {0}/{1}/community + {0}/{1}/testing + + """.format(DEFAULT_MIRROR_URL, alpine_version)) + + self.assertEqual(expected_content, util.load_file(REPO_FILE)) + + def test_main_community_testing_local_repos(self): + """ + Test when details of main, community, testing and + local repos are written to file. + """ + alpine_version = 'v3.12' + local_repo_url = 'http://some.mirror/whereever' + config = { + "apk_repos": { + "alpine_repo": { + "version": alpine_version, + "community_enabled": True, + "testing_enabled": True + }, + "local_repo_base_url": local_repo_url + } + } + + cc_apk_configure.handle(self.name, config, self.cloud, self.log, + self.args) + + expected_content = textwrap.dedent("""\ + # + # Created by cloud-init + # + # This file is written on first boot of an instance + # + + {0}/{1}/main + {0}/{1}/community + # + # Testing - using with non-Edge installation may cause problems! + # + {0}/edge/testing + + # + # Local repo + # + {2}/{1} + + """.format(DEFAULT_MIRROR_URL, alpine_version, local_repo_url)) + + self.assertEqual(expected_content, util.load_file(REPO_FILE)) + + def test_edge_main_community_testing_local_repos(self): + """ + Test when details of main, community, testing and local repos + for Edge version of Alpine are written to file. + """ + alpine_version = 'edge' + local_repo_url = 'http://some.mirror/whereever' + config = { + "apk_repos": { + "alpine_repo": { + "version": alpine_version, + "community_enabled": True, + "testing_enabled": True + }, + "local_repo_base_url": local_repo_url + } + } + + cc_apk_configure.handle(self.name, config, self.cloud, self.log, + self.args) + + expected_content = textwrap.dedent("""\ + # + # Created by cloud-init + # + # This file is written on first boot of an instance + # + + {0}/{1}/main + {0}/{1}/community + {0}/edge/testing + + # + # Local repo + # + {2}/{1} + + """.format(DEFAULT_MIRROR_URL, alpine_version, local_repo_url)) + + self.assertEqual(expected_content, util.load_file(REPO_FILE)) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index c1aff181..e74a0a08 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import cloud +from cloudinit import distros from cloudinit.config import cc_ca_certs from cloudinit import helpers from cloudinit import subp @@ -46,8 +47,9 @@ class TestConfig(TestCase): def setUp(self): super(TestConfig, self).setUp() self.name = "ca-certs" + distro = self._fetch_distro('ubuntu') self.paths = None - self.cloud = cloud.Cloud(None, self.paths, None, None, None) + self.cloud = cloud.Cloud(None, self.paths, None, distro, None) self.log = logging.getLogger("TestNoConfig") self.args = [] @@ -62,6 +64,11 @@ class TestConfig(TestCase): self.mock_remove = self.mocks.enter_context( mock.patch.object(cc_ca_certs, 'remove_default_ca_certs')) + def _fetch_distro(self, kind): + cls = distros.fetch(kind) + paths = helpers.Paths({}) + return cls(kind, {}, paths) + def test_no_trusted_list(self): """ Test that no certificates are written if the 'trusted' key is not @@ -275,7 +282,7 @@ class TestRemoveDefaultCaCerts(TestCase): mock.patch.object(util, 'write_file')) mock_subp = mocks.enter_context(mock.patch.object(subp, 'subp')) - cc_ca_certs.remove_default_ca_certs() + cc_ca_certs.remove_default_ca_certs('ubuntu') mock_delete.assert_has_calls([ mock.call("/usr/share/ca-certificates/"), diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 92a33ec1..6b9c8377 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -239,6 +239,35 @@ class TestNtp(FilesystemMockingTestCase): self.assertEqual(delta[distro][client][key], result[client][key]) + def _get_expected_pools(self, pools, distro, client): + if client in ['ntp', 'chrony']: + if client == 'ntp' and distro == 'alpine': + # NTP for Alpine Linux is Busybox's ntp which does not + # support 'pool' lines in its configuration file. + expected_pools = [] + else: + expected_pools = [ + 'pool {0} iburst'.format(pool) for pool in pools] + elif client == 'systemd-timesyncd': + expected_pools = " ".join(pools) + + return expected_pools + + def _get_expected_servers(self, servers, distro, client): + if client in ['ntp', 'chrony']: + if client == 'ntp' and distro == 'alpine': + # NTP for Alpine Linux is Busybox's ntp which only supports + # 'server' lines without iburst option. + expected_servers = [ + 'server {0}'.format(srv) for srv in servers] + else: + expected_servers = [ + 'server {0} iburst'.format(srv) for srv in servers] + elif client == 'systemd-timesyncd': + expected_servers = " ".join(servers) + + return expected_servers + def test_ntp_handler_real_distro_ntp_templates(self): """Test ntp handler renders the shipped distro ntp client templates.""" pools = ['0.mycompany.pool.ntp.org', '3.mycompany.pool.ntp.org'] @@ -269,27 +298,35 @@ class TestNtp(FilesystemMockingTestCase): content = util.load_file(confpath) if client in ['ntp', 'chrony']: content_lines = content.splitlines() - expected_servers = [ - 'server {0} iburst'.format(srv) for srv in servers] + expected_servers = self._get_expected_servers(servers, + distro, + client) print('distro=%s client=%s' % (distro, client)) for sline in expected_servers: self.assertIn(sline, content_lines, ('failed to render {0} conf' ' for distro:{1}'.format(client, distro))) - expected_pools = [ - 'pool {0} iburst'.format(pool) for pool in pools] - for pline in expected_pools: - self.assertIn(pline, content_lines, - ('failed to render {0} conf' - ' for distro:{1}'.format(client, - distro))) + expected_pools = self._get_expected_pools(pools, distro, + client) + if expected_pools != []: + for pline in expected_pools: + self.assertIn(pline, content_lines, + ('failed to render {0} conf' + ' for distro:{1}'.format(client, + distro))) elif client == 'systemd-timesyncd': + expected_servers = self._get_expected_servers(servers, + distro, + client) + expected_pools = self._get_expected_pools(pools, + distro, + client) expected_content = ( "# cloud-init generated file\n" + "# See timesyncd.conf(5) for details.\n\n" + - "[Time]\nNTP=%s %s \n" % (" ".join(servers), - " ".join(pools))) + "[Time]\nNTP=%s %s \n" % (expected_servers, + expected_pools)) self.assertEqual(expected_content, content) def test_no_ntpcfg_does_nothing(self): @@ -312,10 +349,20 @@ class TestNtp(FilesystemMockingTestCase): confpath = ntpconfig['confpath'] m_select.return_value = ntpconfig cc_ntp.handle('cc_ntp', valid_empty_config, mycloud, None, []) - pools = cc_ntp.generate_server_names(mycloud.distro.name) - self.assertEqual( - "servers []\npools {0}\n".format(pools), - util.load_file(confpath)) + if distro == 'alpine': + # _mock_ntp_client_config call above did not specify a + # client value and so it defaults to "ntp" which on + # Alpine Linux only supports servers and not pools. + + servers = cc_ntp.generate_server_names(mycloud.distro.name) + self.assertEqual( + "servers {0}\npools []\n".format(servers), + util.load_file(confpath)) + else: + pools = cc_ntp.generate_server_names(mycloud.distro.name) + self.assertEqual( + "servers []\npools {0}\n".format(pools), + util.load_file(confpath)) self.assertNotIn('Invalid config:', self.logs.getvalue()) @skipUnlessJsonSchema() @@ -374,18 +421,19 @@ class TestNtp(FilesystemMockingTestCase): invalid_config = { 'ntp': {'invalidkey': 1, 'pools': ['0.mycompany.pool.ntp.org']}} for distro in cc_ntp.distros: - mycloud = self._get_cloud(distro) - ntpconfig = self._mock_ntp_client_config(distro=distro) - confpath = ntpconfig['confpath'] - m_select.return_value = ntpconfig - cc_ntp.handle('cc_ntp', invalid_config, mycloud, None, []) - self.assertIn( - "Invalid config:\nntp: Additional properties are not allowed " - "('invalidkey' was unexpected)", - self.logs.getvalue()) - self.assertEqual( - "servers []\npools ['0.mycompany.pool.ntp.org']\n", - util.load_file(confpath)) + if distro != 'alpine': + mycloud = self._get_cloud(distro) + ntpconfig = self._mock_ntp_client_config(distro=distro) + confpath = ntpconfig['confpath'] + m_select.return_value = ntpconfig + cc_ntp.handle('cc_ntp', invalid_config, mycloud, None, []) + self.assertIn( + "Invalid config:\nntp: Additional properties are not " + "allowed ('invalidkey' was unexpected)", + self.logs.getvalue()) + self.assertEqual( + "servers []\npools ['0.mycompany.pool.ntp.org']\n", + util.load_file(confpath)) @skipUnlessJsonSchema() @mock.patch('cloudinit.config.cc_ntp.select_ntp_client') @@ -452,9 +500,22 @@ class TestNtp(FilesystemMockingTestCase): confpath = ntpconfig['confpath'] service_name = ntpconfig['service_name'] m_select.return_value = ntpconfig - pools = cc_ntp.generate_server_names(mycloud.distro.name) - # force uses systemd path - m_sysd.return_value = True + + hosts = cc_ntp.generate_server_names(mycloud.distro.name) + uses_systemd = True + expected_service_call = ['systemctl', 'reload-or-restart', + service_name] + expected_content = "servers []\npools {0}\n".format(hosts) + + if distro == 'alpine': + uses_systemd = False + expected_service_call = ['service', service_name, 'restart'] + # _mock_ntp_client_config call above did not specify a client + # value and so it defaults to "ntp" which on Alpine Linux only + # supports servers and not pools. + expected_content = "servers {0}\npools []\n".format(hosts) + + m_sysd.return_value = uses_systemd with mock.patch('cloudinit.config.cc_ntp.util') as m_util: # allow use of util.mergemanydict m_util.mergemanydict.side_effect = util.mergemanydict @@ -465,11 +526,9 @@ class TestNtp(FilesystemMockingTestCase): cfg['ntp']['enabled']) cc_ntp.handle('notimportant', cfg, mycloud, None, None) m_subp.subp.assert_called_with( - ['systemctl', 'reload-or-restart', - service_name], capture=True) - self.assertEqual( - "servers []\npools {0}\n".format(pools), - util.load_file(confpath)) + expected_service_call, capture=True) + + self.assertEqual(expected_content, util.load_file(confpath)) def test_opensuse_picks_chrony(self): """Test opensuse picks chrony or ntp on certain distro versions""" diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 0d8d17b9..93b24fdc 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -11,62 +11,63 @@ from cloudinit.tests.helpers import mock class TestLoadPowerState(t_help.TestCase): def test_no_config(self): # completely empty config should mean do nothing - (cmd, _timeout, _condition) = psc.load_power_state({}) + (cmd, _timeout, _condition) = psc.load_power_state({}, 'ubuntu') self.assertIsNone(cmd) def test_irrelevant_config(self): # no power_state field in config should return None for cmd - (cmd, _timeout, _condition) = psc.load_power_state({'foo': 'bar'}) + (cmd, _timeout, _condition) = psc.load_power_state({'foo': 'bar'}, + 'ubuntu') self.assertIsNone(cmd) def test_invalid_mode(self): cfg = {'power_state': {'mode': 'gibberish'}} - self.assertRaises(TypeError, psc.load_power_state, cfg) + self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') cfg = {'power_state': {'mode': ''}} - self.assertRaises(TypeError, psc.load_power_state, cfg) + self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') def test_empty_mode(self): cfg = {'power_state': {'message': 'goodbye'}} - self.assertRaises(TypeError, psc.load_power_state, cfg) + self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') def test_valid_modes(self): cfg = {'power_state': {}} for mode in ('halt', 'poweroff', 'reboot'): cfg['power_state']['mode'] = mode - check_lps_ret(psc.load_power_state(cfg), mode=mode) + check_lps_ret(psc.load_power_state(cfg, 'ubuntu'), mode=mode) def test_invalid_delay(self): cfg = {'power_state': {'mode': 'poweroff', 'delay': 'goodbye'}} - self.assertRaises(TypeError, psc.load_power_state, cfg) + self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') def test_valid_delay(self): cfg = {'power_state': {'mode': 'poweroff', 'delay': ''}} for delay in ("now", "+1", "+30"): cfg['power_state']['delay'] = delay - check_lps_ret(psc.load_power_state(cfg)) + check_lps_ret(psc.load_power_state(cfg, 'ubuntu')) def test_message_present(self): cfg = {'power_state': {'mode': 'poweroff', 'message': 'GOODBYE'}} - ret = psc.load_power_state(cfg) - check_lps_ret(psc.load_power_state(cfg)) + ret = psc.load_power_state(cfg, 'ubuntu') + check_lps_ret(psc.load_power_state(cfg, 'ubuntu')) self.assertIn(cfg['power_state']['message'], ret[0]) def test_no_message(self): # if message is not present, then no argument should be passed for it cfg = {'power_state': {'mode': 'poweroff'}} - (cmd, _timeout, _condition) = psc.load_power_state(cfg) + (cmd, _timeout, _condition) = psc.load_power_state(cfg, 'ubuntu') self.assertNotIn("", cmd) - check_lps_ret(psc.load_power_state(cfg)) + check_lps_ret(psc.load_power_state(cfg, 'ubuntu')) self.assertTrue(len(cmd) == 3) def test_condition_null_raises(self): cfg = {'power_state': {'mode': 'poweroff', 'condition': None}} - self.assertRaises(TypeError, psc.load_power_state, cfg) + self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') def test_condition_default_is_true(self): cfg = {'power_state': {'mode': 'poweroff'}} - _cmd, _timeout, cond = psc.load_power_state(cfg) + _cmd, _timeout, cond = psc.load_power_state(cfg, 'ubuntu') self.assertEqual(cond, True) diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 99f0b06c..44292571 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -24,6 +24,7 @@ class GetSchemaTest(CiTestCase): schema = get_schema() self.assertCountEqual( [ + 'cc_apk_configure', 'cc_apt_configure', 'cc_bootcmd', 'cc_locale', diff --git a/tools/render-cloudcfg b/tools/render-cloudcfg index 9322b2c3..ed454840 100755 --- a/tools/render-cloudcfg +++ b/tools/render-cloudcfg @@ -4,8 +4,9 @@ import argparse import os import sys -VARIANTS = ["amazon", "arch", "centos", "debian", "fedora", "freebsd", - "netbsd", "openbsd", "rhel", "suse", "ubuntu", "unknown"] +VARIANTS = ["alpine", "amazon", "arch", "centos", "debian", "fedora", + "freebsd", "netbsd", "openbsd", "rhel", "suse", "ubuntu", + "unknown"] if "avoid-pep8-E402-import-not-top-of-file": -- cgit v1.2.3 From c73ab5665469a28dec2995b2b15d3462b0a83c4b Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Fri, 21 Aug 2020 18:23:24 +0200 Subject: Update the list of valid ssh keys. (#487) Update ssh_util.py with latest list of keys (from openssh-8.3p1/sshkey.c), Added keys: sk-ecdsa-sha2-nistp256-cert-v01@openssh.com sk-ecdsa-sha2-nistp256@openssh.com sk-ssh-ed25519-cert-v01@openssh.com sk-ssh-ed25519@openssh.com ssh-xmss-cert-v01@openssh.com ssh-xmss@openssh.com LP: #1877869 --- cloudinit/config/cc_ssh.py | 36 ++++ cloudinit/config/cc_ssh_authkey_fingerprints.py | 6 +- cloudinit/ssh_util.py | 40 ++-- tests/unittests/test_sshutil.py | 251 ++++++++++++++++++++++-- tools/.github-cla-signers | 1 + 5 files changed, 299 insertions(+), 35 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 228e5e0d..9b2a333a 100755 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -35,6 +35,42 @@ root login is disabled, and root login opts are set to:: no-port-forwarding,no-agent-forwarding,no-X11-forwarding +Supported public key types for the ``ssh_authorized_keys`` are: + + - dsa + - rsa + - ecdsa + - ed25519 + - ecdsa-sha2-nistp256-cert-v01@openssh.com + - ecdsa-sha2-nistp256 + - ecdsa-sha2-nistp384-cert-v01@openssh.com + - ecdsa-sha2-nistp384 + - ecdsa-sha2-nistp521-cert-v01@openssh.com + - ecdsa-sha2-nistp521 + - sk-ecdsa-sha2-nistp256-cert-v01@openssh.com + - sk-ecdsa-sha2-nistp256@openssh.com + - sk-ssh-ed25519-cert-v01@openssh.com + - sk-ssh-ed25519@openssh.com + - ssh-dss-cert-v01@openssh.com + - ssh-dss + - ssh-ed25519-cert-v01@openssh.com + - ssh-ed25519 + - ssh-rsa-cert-v01@openssh.com + - ssh-rsa + - ssh-xmss-cert-v01@openssh.com + - ssh-xmss@openssh.com + +.. note:: + this list has been filtered out from the supported keytypes of + `OpenSSH`_ source, where the sigonly keys are removed. Please see + ``ssh_util`` for more information. + + ``dsa``, ``rsa``, ``ecdsa`` and ``ed25519`` are added for legacy, + as they are valid public keys in some old distros. They can possibly + be removed in the future when support for the older distros are dropped + +.. _OpenSSH: https://github.com/openssh/openssh-portable/blob/master/sshkey.c + Host Keys ^^^^^^^^^ diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 7ac1c8cf..05d30ad1 100755 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -13,7 +13,7 @@ Write fingerprints of authorized keys for each user to log. This is enabled by default, but can be disabled using ``no_ssh_fingerprints``. The hash type for the keys can be specified, but defaults to ``sha256``. -**Internal name:** `` cc_ssh_authkey_fingerprints`` +**Internal name:** ``cc_ssh_authkey_fingerprints`` **Module frequency:** per instance @@ -59,8 +59,8 @@ def _gen_fingerprint(b64_text, hash_meth='sha256'): def _is_printable_key(entry): if any([entry.keytype, entry.base64, entry.comment, entry.options]): - if (entry.keytype and - entry.keytype.lower().strip() in ['ssh-dss', 'ssh-rsa']): + if (entry.keytype and entry.keytype.lower().strip() + in ssh_util.VALID_KEY_TYPES): return True return False diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 89150acf..c08042d6 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -17,27 +17,43 @@ LOG = logging.getLogger(__name__) # See: man sshd_config DEF_SSHD_CFG = "/etc/ssh/sshd_config" -# taken from OpenSSH source openssh-7.3p1/sshkey.c: -# static const struct keytype keytypes[] = { ... } +# this list has been filtered out from keytypes of OpenSSH source +# openssh-8.3p1/sshkey.c: +# static const struct keytype keytypes[] = { +# filter out the keytypes with the sigonly flag, eg: +# { "rsa-sha2-256", "RSA", NULL, KEY_RSA, 0, 0, 1 }, +# refer to the keytype struct of OpenSSH in the same file, to see +# if the position of the sigonly flag has been moved. +# +# dsa, rsa, ecdsa and ed25519 are added for legacy, as they are valid +# public keys in some old distros. They can possibly be removed +# in the future when support for the older distros is dropped +# +# When updating the list, also update the _is_printable_key list in +# cloudinit/config/cc_ssh_authkey_fingerprints.py VALID_KEY_TYPES = ( "dsa", + "rsa", "ecdsa", - "ecdsa-sha2-nistp256", + "ed25519", "ecdsa-sha2-nistp256-cert-v01@openssh.com", - "ecdsa-sha2-nistp384", + "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384-cert-v01@openssh.com", - "ecdsa-sha2-nistp521", + "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521-cert-v01@openssh.com", - "ed25519", - "rsa", - "rsa-sha2-256", - "rsa-sha2-512", - "ssh-dss", + "ecdsa-sha2-nistp521", + "sk-ecdsa-sha2-nistp256-cert-v01@openssh.com", + "sk-ecdsa-sha2-nistp256@openssh.com", + "sk-ssh-ed25519-cert-v01@openssh.com", + "sk-ssh-ed25519@openssh.com", "ssh-dss-cert-v01@openssh.com", - "ssh-ed25519", + "ssh-dss", "ssh-ed25519-cert-v01@openssh.com", - "ssh-rsa", + "ssh-ed25519", "ssh-rsa-cert-v01@openssh.com", + "ssh-rsa", + "ssh-xmss-cert-v01@openssh.com", + "ssh-xmss@openssh.com", ) _DISABLE_USER_SSH_EXIT = 142 diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index d15fc60b..fd1d1bac 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -15,6 +15,9 @@ FakePwEnt.__new__.__defaults__ = tuple( "UNSET_%s" % n for n in FakePwEnt._fields) +# Do not use these public keys, most of them are fetched from +# the testdata for OpenSSH, and their private keys are available +# https://github.com/openssh/openssh-portable/tree/master/regress/unittests/sshkey/testdata VALID_CONTENT = { 'dsa': ( "AAAAB3NzaC1kc3MAAACBAIrjOQSlSea19bExXBMBKBvcLhBoVvNBjCppNzllipF" @@ -41,24 +44,238 @@ VALID_CONTENT = { "YWpMfYdPUnE7u536WqzFmsaqJctz3gBxH9Ex7dFtrxR4qiqEr9Qtlu3xGn7Bw07" "/+i1D+ey3ONkZLN+LQ714cgj8fRS4Hj29SCmXp5Kt5/82cD/VN3NtHw==" ), + 'ed25519': ( + "AAAAC3NzaC1lZDI1NTE5AAAAIA1J77+CrJ8p6/vWCEzuylqJNMHUP/XmeYyGVWb" + "8lnDd" + ), + 'ecdsa-sha2-nistp256-cert-v01@openssh.com': ( + "AAAAKGVjZHNhLXNoYTItbmlzdHAyNTYtY2VydC12MDFAb3BlbnNzaC5jb20AAAA" + "gQIfwT/+UX68/hlKsdKuaOuAVB6ftTg03SlP/uH4OBEwAAAAIbmlzdHAyNTYAAA" + "BBBEjA0gjJmPM6La3sXyfNlnjilvvGY6I2M8SvJj4o3X/46wcUbPWTaj4RF3EXw" + "HvNxplYBwdPlk2zEecvf9Cs2BMAAAAAAAAAAAAAAAEAAAAYa2V5cy9lY2RzYS1z" + "aGEyLW5pc3RwMjU2AAAAAAAAAAAAAAAA//////////8AAAAAAAAAggAAABVwZXJ" + "taXQtWDExLWZvcndhcmRpbmcAAAAAAAAAF3Blcm1pdC1hZ2VudC1mb3J3YXJkaW" + "5nAAAAAAAAABZwZXJtaXQtcG9ydC1mb3J3YXJkaW5nAAAAAAAAAApwZXJtaXQtc" + "HR5AAAAAAAAAA5wZXJtaXQtdXNlci1yYwAAAAAAAAAAAAAAaAAAABNlY2RzYS1z" + "aGEyLW5pc3RwMjU2AAAACG5pc3RwMjU2AAAAQQRH6Y9Q1+ocQ8ETKW3LjQqtxg7" + "OuSSDacxmmQatQVaIawwjCbmntyEAqmVj3v9ElDSXnO5m7TyYMBQu4+vsh76RAA" + "AAZQAAABNlY2RzYS1zaGEyLW5pc3RwMjU2AAAASgAAACEA47Cl2MMhr+glPGuxx" + "2tM3QXkDcwdP0SxSEW5yy4XV5oAAAAhANNMm1cdVlAt3hmycQgdD82zPlg5YvVO" + "iN0SQTbgVD8i" + ), 'ecdsa-sha2-nistp256': ( - "AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMy/WuXq5MF" - "r5hVQ9EEKKUTF7vUaOkgxUh6bNsCs9SFMVslIm1zM/WJYwUv52LdEePjtDYiV4A" - "l2XthJ9/bs7Pc=" + "AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEjA0gjJmPM" + "6La3sXyfNlnjilvvGY6I2M8SvJj4o3X/46wcUbPWTaj4RF3EXwHvNxplYBwdPlk" + "2zEecvf9Cs2BM=" ), - 'ecdsa-sha2-nistp521': ( - "AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBABOdNTkh9F" - "McK4hZRLs5LTXBEXwNr0+Yg9uvJYRFcz2ZlnjYX9tM4Z3QQFjqogU4pU+zpKLqZ" - "5VE4Jcnb1T608UywBIdXkSFZT8trGJqBv9nFWGgmTX3KP8kiBbihpuv1cGwglPl" - "Hxs50A42iP0JiT7auGtEAGsu/uMql323GTGb4171Q==" + 'ecdsa-sha2-nistp384-cert-v01@openssh.com': ( + "AAAAKGVjZHNhLXNoYTItbmlzdHAzODQtY2VydC12MDFAb3BlbnNzaC5jb20AAAA" + "grnSvDsK1EnCZndO1IyGWcGkVgVSkPWi/XO2ybPFyLVUAAAAIbmlzdHAzODQAAA" + "BhBAaYSQs+8TT0Tzciy0dorwhur6yzOGUrYQ6ueUQYWbE7eNdHmhsVrlpGPgSaY" + "ByhXtAJiPOMqLU5h0eb3sCtM3ek4NvjXFTGTqPrrxJI6q0OsgrtkGE7UM9ZsfMm" + "7q6BOAAAAAAAAAAAAAAAAQAAABhrZXlzL2VjZHNhLXNoYTItbmlzdHAzODQAAAA" + "AAAAAAAAAAAD//////////wAAAAAAAACCAAAAFXBlcm1pdC1YMTEtZm9yd2FyZG" + "luZwAAAAAAAAAXcGVybWl0LWFnZW50LWZvcndhcmRpbmcAAAAAAAAAFnBlcm1pd" + "C1wb3J0LWZvcndhcmRpbmcAAAAAAAAACnBlcm1pdC1wdHkAAAAAAAAADnBlcm1p" + "dC11c2VyLXJjAAAAAAAAAAAAAACIAAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAA" + "IbmlzdHAzODQAAABhBLWbubcMzcWc7lMTCMGVXZlaVvUOHLjpr6SOOScFFrd8K9" + "Gl8nYELST5HZ1gym65m+MG6/tbrUWIY/flLWNIe+WtqxrdPPGdIhFruCwNw2peZ" + "SbQOa/o3AGnJ/vO6EKEGAAAAIQAAAATZWNkc2Etc2hhMi1uaXN0cDM4NAAAAGkA" + "AAAxAL10JHd5bvnbpD+fet/k1YE1BEIrqGXaoIIJ9ReE5H4nTK1uQJzMD7+wwGK" + "RVYqYQgAAADAiit0UCMDAUbjD+R2x4LvU3x/t8G3sdqDLRNfMRpjZpvcS8AwC+Y" + "VFVSQNn0AyzW0=" ), 'ecdsa-sha2-nistp384': ( - "AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBAnoqFU9Gnl" - "LcsEuCJnobs/c6whzvjCgouaOO61kgXNtIxyF4Wkutg6xaGYgBBt/phb7a2TurI" - "bcIBuzJ/mP22UyUAbNnBfStAEBmYbrTf1EfiMCYUAr1XnL0UdYmZ8HFg==" + "AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBAaYSQs+8TT" + "0Tzciy0dorwhur6yzOGUrYQ6ueUQYWbE7eNdHmhsVrlpGPgSaYByhXtAJiPOMqL" + "U5h0eb3sCtM3ek4NvjXFTGTqPrrxJI6q0OsgrtkGE7UM9ZsfMm7q6BOA==" + ), + 'ecdsa-sha2-nistp521-cert-v01@openssh.com': ( + "AAAAKGVjZHNhLXNoYTItbmlzdHA1MjEtY2VydC12MDFAb3BlbnNzaC5jb20AAAA" + "gGmRzkkMvRFk1V5U3m3mQ2nfW20SJVXk1NKnT5iZGDcEAAAAIbmlzdHA1MjEAAA" + "CFBAHosAOHAI1ZkerbKYQ72S6uit1u77PCj/OalZtXgsxv0TTAZB273puG2X94C" + "Q8yyNHcby87zFZHdv5BSKyZ/cyREAAeiAcSakop9VS3+bUfZpEIqwBZXarwUjnR" + "nxprkcQ0rfCCdagkGZr/OA7DemK2D8tKLTHsKoEEWNImo6/pXDkFxAAAAAAAAAA" + "AAAAAAQAAABhrZXlzL2VjZHNhLXNoYTItbmlzdHA1MjEAAAAAAAAAAAAAAAD///" + "///////wAAAAAAAACCAAAAFXBlcm1pdC1YMTEtZm9yd2FyZGluZwAAAAAAAAAXc" + "GVybWl0LWFnZW50LWZvcndhcmRpbmcAAAAAAAAAFnBlcm1pdC1wb3J0LWZvcndh" + "cmRpbmcAAAAAAAAACnBlcm1pdC1wdHkAAAAAAAAADnBlcm1pdC11c2VyLXJjAAA" + "AAAAAAAAAAACsAAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAA" + "CFBAC6hFVXM1XEg/7qKkp5sLZuANGQVW88b5pPn2ZcK0td9IQstLH6BwWuZ6MPE" + "ogiDlvx9HD1BaKGBBfkxgOY8NGFzQHbjU9eTWH3gt0RATDbZsij1pSkFPnAXdU9" + "SjfogYloI2xdHaTCgWp3zgsUV+BBQ0QGGv2MqqcOmrF0f5YEJeOffAAAAKcAAAA" + "TZWNkc2Etc2hhMi1uaXN0cDUyMQAAAIwAAABCAT+vSOYPuYVTDopDW08576d5Sb" + "edXQMOu1op4CQIm98VKtAXvu5dfioi5VYAqpte8M+UxEMOMiQWJp+U9exYf6LuA" + "AAAQgEzkIpX3yKXPaPcK17mNx40ujEDitm4ARmbhAge0sFhZtf7YIgI55b6vkI8" + "JvMJkzQCBF1cpNOaIpVh1nFZNBphMQ==" + ), + 'ecdsa-sha2-nistp521': ( + "AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAHosAOHAI1" + "ZkerbKYQ72S6uit1u77PCj/OalZtXgsxv0TTAZB273puG2X94CQ8yyNHcby87zF" + "ZHdv5BSKyZ/cyREAAeiAcSakop9VS3+bUfZpEIqwBZXarwUjnRnxprkcQ0rfCCd" + "agkGZr/OA7DemK2D8tKLTHsKoEEWNImo6/pXDkFxA==" + ), + 'sk-ecdsa-sha2-nistp256-cert-v01@openssh.com': ( + "AAAAIHNzaC1lZDI1NTE5LWNlcnQtdjAxQG9wZW5zc2guY29tAAAAIIxzuxl4z3u" + "wAIslne8Huft+1n1IhHAlNbWZkQyyECCGAAAAIFOG6kY7Rf4UtCFvPwKgo/BztX" + "ck2xC4a2WyA34XtIwZAAAAAAAAAAgAAAACAAAABmp1bGl1cwAAABIAAAAFaG9zd" + "DEAAAAFaG9zdDIAAAAANowB8AAAAABNHmBwAAAAAAAAAAAAAAAAAAAAMwAAAAtz" + "c2gtZWQyNTUxOQAAACBThupGO0X+FLQhbz8CoKPwc7V3JNsQuGtlsgN+F7SMGQA" + "AAFMAAAALc3NoLWVkMjU1MTkAAABABGTn+Bmz86Ajk+iqKCSdP5NClsYzn4alJd" + "0V5bizhP0Kumc/HbqQfSt684J1WdSzih+EjvnTgBhK9jTBKb90AQ==" + ), + 'sk-ecdsa-sha2-nistp256@openssh.com': ( + "AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHA" + "yNTYAAABBBIELQJ2DgvaX1yQlKFokfWM2suuaCFI2qp0eJodHyg6O4ifxc3XpRK" + "d1OS8dNYQtE/YjdXSrA+AOnMF5ns2Nkx4AAAAEc3NoOg==" + ), + 'sk-ssh-ed25519-cert-v01@openssh.com': ( + "AAAAIHNzaC1lZDI1NTE5LWNlcnQtdjAxQG9wZW5zc2guY29tAAAAIIxzuxl4z3u" + "wAIslne8Huft+1n1IhHAlNbWZkQyyECCGAAAAIFOG6kY7Rf4UtCFvPwKgo/BztX" + "ck2xC4a2WyA34XtIwZAAAAAAAAAAgAAAACAAAABmp1bGl1cwAAABIAAAAFaG9zd" + "DEAAAAFaG9zdDIAAAAANowB8AAAAABNHmBwAAAAAAAAAAAAAAAAAAAAMwAAAAtz" + "c2gtZWQyNTUxOQAAACBThupGO0X+FLQhbz8CoKPwc7V3JNsQuGtlsgN+F7SMGQA" + "AAFMAAAALc3NoLWVkMjU1MTkAAABABGTn+Bmz86Ajk+iqKCSdP5NClsYzn4alJd" + "0V5bizhP0Kumc/HbqQfSt684J1WdSzih+EjvnTgBhK9jTBKb90AQ==" + ), + 'sk-ssh-ed25519@openssh.com': ( + "AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAICFo/k5LU8863u66YC9" + "eUO2170QduohPURkQnbLa/dczAAAABHNzaDo=" + ), + 'ssh-dss-cert-v01@openssh.com': ( + "AAAAHHNzaC1kc3MtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgdTlbNU9Hn9Qng3F" + "HxwH971bxCIoq1ern/QWFFDWXgmYAAACBAPqS600VGwdPAQC/p3f0uGyrLVql0c" + "Fn1zYd/JGvtabKnIYjLaYprje/NcjwI3CZFJiz4Dp3S8kLs+X5/1DMn/Tg1Y4D4" + "yLB+6vCtHcJF7rVBFhvw/KZwc7G54ez3khyOtsg82fzpyOc8/mq+/+C5TMKO7DD" + "jMF0k5emWKCsa3ZfAAAAFQCjA/+dKkMu4/CWjJPtfl7YNaStNQAAAIEA7uX1BVV" + "tJKjLmWrpw62+l/xSXA5rr7MHBuWjiCYV3VHBfXJaQDyRDtGuEJKDwdzqYgacpG" + "ApGWL/cuBtJ9nShsUl6GRG0Ra03g+Hx9VR5LviJBsjAVB4qVgciU1NGga0Bt2Le" + "cd1X4EGQRBzVXeuOpiqGM6jP/I2yDMs0Pboet0AAACBAOdXpyfmobEBaOqZAuvg" + "j1P0uhjG2P31Ufurv22FWPBU3A9qrkxbOXwE0LwvjCvrsQV/lrYhJz/tiys40Ve" + "ahulWZE5SAHMXGIf95LiLSgaXMjko7joot+LK84ltLymwZ4QMnYjnZSSclf1Uuy" + "QMcUtb34+I0u9Ycnyhp2mSFsQtAAAAAAAAAAYAAAACAAAABmp1bGl1cwAAABIAA" + "AAFaG9zdDEAAAAFaG9zdDIAAAAANowB8AAAAABNHmBwAAAAAAAAAAAAAAAAAAAA" + "MwAAAAtzc2gtZWQyNTUxOQAAACBThupGO0X+FLQhbz8CoKPwc7V3JNsQuGtlsgN" + "+F7SMGQAAAFMAAAALc3NoLWVkMjU1MTkAAABAh/z1LIdNL1b66tQ8t9DY9BTB3B" + "QKpTKmc7ezyFKLwl96yaIniZwD9Ticdbe/8i/Li3uCFE3EAt8NAIv9zff8Bg==" + ), + 'ssh-dss': ( + "AAAAB3NzaC1kc3MAAACBAPqS600VGwdPAQC/p3f0uGyrLVql0cFn1zYd/JGvtab" + "KnIYjLaYprje/NcjwI3CZFJiz4Dp3S8kLs+X5/1DMn/Tg1Y4D4yLB+6vCtHcJF7" + "rVBFhvw/KZwc7G54ez3khyOtsg82fzpyOc8/mq+/+C5TMKO7DDjMF0k5emWKCsa" + "3ZfAAAAFQCjA/+dKkMu4/CWjJPtfl7YNaStNQAAAIEA7uX1BVVtJKjLmWrpw62+" + "l/xSXA5rr7MHBuWjiCYV3VHBfXJaQDyRDtGuEJKDwdzqYgacpGApGWL/cuBtJ9n" + "ShsUl6GRG0Ra03g+Hx9VR5LviJBsjAVB4qVgciU1NGga0Bt2Lecd1X4EGQRBzVX" + "euOpiqGM6jP/I2yDMs0Pboet0AAACBAOdXpyfmobEBaOqZAuvgj1P0uhjG2P31U" + "furv22FWPBU3A9qrkxbOXwE0LwvjCvrsQV/lrYhJz/tiys40VeahulWZE5SAHMX" + "GIf95LiLSgaXMjko7joot+LK84ltLymwZ4QMnYjnZSSclf1UuyQMcUtb34+I0u9" + "Ycnyhp2mSFsQt" + ), + 'ssh-ed25519-cert-v01@openssh.com': ( + "AAAAIHNzaC1lZDI1NTE5LWNlcnQtdjAxQG9wZW5zc2guY29tAAAAIIxzuxl4z3u" + "wAIslne8Huft+1n1IhHAlNbWZkQyyECCGAAAAIFOG6kY7Rf4UtCFvPwKgo/BztX" + "ck2xC4a2WyA34XtIwZAAAAAAAAAAgAAAACAAAABmp1bGl1cwAAABIAAAAFaG9zd" + "DEAAAAFaG9zdDIAAAAANowB8AAAAABNHmBwAAAAAAAAAAAAAAAAAAAAMwAAAAtz" + "c2gtZWQyNTUxOQAAACBThupGO0X+FLQhbz8CoKPwc7V3JNsQuGtlsgN+F7SMGQA" + "AAFMAAAALc3NoLWVkMjU1MTkAAABABGTn+Bmz86Ajk+iqKCSdP5NClsYzn4alJd" + "0V5bizhP0Kumc/HbqQfSt684J1WdSzih+EjvnTgBhK9jTBKb90AQ==" + ), + 'ssh-ed25519': ( + "AAAAC3NzaC1lZDI1NTE5AAAAIFOG6kY7Rf4UtCFvPwKgo/BztXck2xC4a2WyA34" + "XtIwZ" + ), + 'ssh-rsa-cert-v01@openssh.com': ( + "AAAAHHNzaC1yc2EtY2VydC12MDFAb3BlbnNzaC5jb20AAAAg98LhS2EHxLOWCLo" + "pZPwHdg/RJXusnkOqQXSc9R7aITkAAAADAQABAAAAgQDLV5lUTt7FrADseB/CGh" + "EZzpoojjEW5y8+ePvLppmK3MmMI18ud6vxzpK3bwZLYkVSyfJYI0HmIuGhdu7yM" + "rW6wb84gbq8C31Xoe9EORcIUuGSvDKdNSM1SjlhDquRblDFB8kToqXyx1lqrXec" + "XylxIUOL0jE+u0rU1967pDJx+wAAAAAAAAAFAAAAAgAAAAZqdWxpdXMAAAASAAA" + "ABWhvc3QxAAAABWhvc3QyAAAAADaMAfAAAAAATR5gcAAAAAAAAAAAAAAAAAAAAD" + "MAAAALc3NoLWVkMjU1MTkAAAAgU4bqRjtF/hS0IW8/AqCj8HO1dyTbELhrZbIDf" + "he0jBkAAABTAAAAC3NzaC1lZDI1NTE5AAAAQI3QGlUCzC07KorupxpDkkGy6tni" + "aZ8EvBflzvv+itXWNchGvfUeHmVT6aX0sRqehdz/lR+GmXRoZBhofwh0qAM=" + ), + 'ssh-rsa': ( + "AAAAB3NzaC1yc2EAAAADAQABAAAAgQDLV5lUTt7FrADseB/CGhEZzpoojjEW5y8" + "+ePvLppmK3MmMI18ud6vxzpK3bwZLYkVSyfJYI0HmIuGhdu7yMrW6wb84gbq8C3" + "1Xoe9EORcIUuGSvDKdNSM1SjlhDquRblDFB8kToqXyx1lqrXecXylxIUOL0jE+u" + "0rU1967pDJx+w==" + ), + 'ssh-xmss-cert-v01@openssh.com': ( + "AAAAHXNzaC14bXNzLWNlcnQtdjAxQG9wZW5zc2guY29tAAAAIM2UD0IH+Igsekq" + "xjTO5f36exX4WGRMCtDGPjwfbXblxAAAAFVhNU1NfU0hBMi0yNTZfVzE2X0gxMA" + "AAAEDI83/K5JMOy0BMJgQypRdz35ApAnoQinMJ8ZMoZPaEJF8Z4rANQlfzaAXum" + "N3RDU5CGIUGGw+WJ904G/wwEq9CAAAAAAAAAAAAAAABAAAACWtleXMveG1zcwAA" + "AAAAAAAAAAAAAP//////////AAAAAAAAAIIAAAAVcGVybWl0LVgxMS1mb3J3YXJ" + "kaW5nAAAAAAAAABdwZXJtaXQtYWdlbnQtZm9yd2FyZGluZwAAAAAAAAAWcGVybW" + "l0LXBvcnQtZm9yd2FyZGluZwAAAAAAAAAKcGVybWl0LXB0eQAAAAAAAAAOcGVyb" + "Wl0LXVzZXItcmMAAAAAAAAAAAAAAHUAAAAUc3NoLXhtc3NAb3BlbnNzaC5jb20A" + "AAAVWE1TU19TSEEyLTI1Nl9XMTZfSDEwAAAAQA+irIyT2kaOd07YWZT/QItzNBZ" + "kUYwnqZJihQ7BxuyiDP4HEFbnfYnnIZXx9Asyi7vDyZRvi+AMSOzmMSq4JnkAAA" + "ngAAAAFHNzaC14bXNzQG9wZW5zc2guY29tAAAJxAAAAAAFjaKTDc+7Hu2uFGIab" + "3NAku8HbbGtrq/uGXOxmqxu4RaLqmwofl5iXk3nMwWEhQAb99vAc9D9ZFtfxJO4" + "STYUTjbj4BxToov/uvbYfE5VeO6sMvkGglgh9YHkCTAItsG8EmGT1SIPfKYzLlN" + "jvUlbcv0PaPFMJ0wzS9mNfuRf+KUhf3dxQ6zaMrBH3KEJ8Me2kNjhnh6rNPROeI" + "N+IcStSKsydYuiySGKS/orsH38XysuK5QqLizbHJY3cqLbkW9LsIijb+pfEJh4Y" + "bOoAbraWAv9ySnWCyRhvw2x8uJ0ZM+p5WSRiZfB3JxCpOhHgiKa9TdmdjnAtnED" + "zqKOj/gM7y9mesn5ydQI0bENOGymlw0ThUGKbXMxn87Hc9dDPURUBmoO3NGjPDf" + "7meS39A1ZEGtCe/pbZU9iwxqGx4wJYvB4lutRP2tYC1pA6hjQCcHibvxl5iqj+1" + "jRjwPr8dbTm4PdETW/7JDSVQXKjxOT0kRLHLelJNeviGx5zSHR5PtnUP3nOBMme" + "hk9DwcQW9vfKeWSnu9CMnF8xvYJxoPKQwmz0TKo+YVOUnc9/Ma+Ykseof9/W+rk" + "USQGELc4x7XE5XBKYZZP2PmtxirQ3qTWFw+CeTX2Oa+jPYkzOa7jgmHJ3Fi9Xqw" + "3L844vRl97e28GmwS0M1SXH+ohES0mO4EcrGh5OLyXBaRTV5QMo+4Bg6FH/HwEn" + "gG1mdEOAqvctK2QC70c4lHGzfexqwQ2U6WUADPcd/BLOE8Noj1EiXYwZrSA1okZ" + "FYnS/b89Uo51D2FE4A33V4gcxAglGzVNtrPulkguNT9B4jjNgdIwkTBL9k3ujkG" + "og6pyYjZ0J5Jp5XPBn+y0LqrpOdZijzrc1OJbX59tTeIbDkM7Fw8As4a03hQPDU" + "FTOdyMHgLnuLhLXOcqIjvW5axZL/Kx3UET8wrSHizPoa6NErCG4v5mC2M4kBSOW" + "In1QV27QMaHkL/ZAa3mPsW5iFZtOVEGzw2BW4MZs0qOrcloCENZzOHiMBroKEkH" + "AbzX6D1FLwml2JpXq4JXlCrdIiFm4+co5ygnWPqb4QGzMlcbjW/x/A16TthNuok" + "wwlmK5ndKZ76LahyGKEwx2Nv0D+0xilEC1EldtiYRdBNlcGbU/A5EhH5bQ9KVIH" + "wjWm35pRPLl5224//nqvQKhwFCn9otsR35XHXev3IQ0or3HmQxIvSDOwir1l66z" + "FFrkyHMWexoucbTBxw1MN3hLb247lcVYJ5+hspJgyoYbfR5RkQVDzhpzskogP7l" + "K5t0bphu+f+hpvrca7DAiiIZkcR4R1UUQoRnJPRXyXOxlxwS10b51cP9p9jzvZj" + "d2LUs8yx1KXWSxNHo6WmtYONNaUfdX2OB5+QCvPULfLfFeBrqpX6Yp5wQMM5Cup" + "k8FEfV07eEgQkVE9nDGKHglWo3kUdOF+XCqWAnXn0b/2bNS9/SSAz6gB1GTFcN/" + "QsFGlC0QgbCJbQ7LQM6hilRWupWvN5zZ/+HJyyRHuSs5VnQnKiGbIa6AIhx7mP7" + "8T82gKjU3mHLJWMGKcT3cY8R958Gs+w4OT71VJRMw3kK6qk02WCbD5OtbFeC6ib" + "KRJKdLK3BzjVs/Fzu3mHVucVby3jpvG1Z8HKspKFhvV7gjFEPu8qHKi4MdAlif/" + "KakyPk8yZB/dMfaxh7Kv/WpJuSwWNs7RNh29e+ZG+POxqRPWiHqiVw7P17a4dN7" + "nkVOawdBEyxI4NAY+4zW+0r0bAy6zNBitBvkq3IXfr3De6Upex52sPHvK04PXoV" + "RI6gjnpPSbLLjpSpcHPKgB7DWefLfhd63BUQbc57D8zm8Jd6qtmzcSKn+wz5/zT" + "0I6v9I4a+DOjjyqpPpzzNU76pt+Y8SuBgHzMm1vcAdNWlbQrqtScvm0T9AkYni6" + "47vSh77uwRZKDtMCMSU151tVUavXhtLYLZ6/ll5NhMXkkx8//i7pk1OBjN5LHVQ" + "0QeimRmavlXU1dJ2rwsFAV+9dDdJXUNOq3VLTo9FrbOzZiWtzzjkJpVJAFREnBn" + "yIDBK5AXtXE1RzfzaBHzbI2e2kO3t+CSNLWYMFYHBDqaeICYQ9+I9aO/8hnzVSo" + "fp+8IfWO8iJhppqynUniicW2oCzrn4oczzYNEjImt8CGY7g90GxWfX+ZgXMJfy/" + "bQiFQL3dZvVypDHEbFoIGz+sxkL83xrP4MZV1V9Wwa64lDXYv01Kp4kQXmmnAZY" + "KlxBoWqYDXLeLLguSOZxDSCIDpd+YPm39wQ3wOysHW2fmsWtp6FPPlQRUYjsGIP" + "lfrkJzpoeaPKDtF1m+mOULfEh9kvTKCmKRi385T9ON39D97eWqaM4CCfUGImvdR" + "DlZLXvjmaAh5BVJ8VJxk75OkP14vWFFlTMv0/k4BYLDKsrNqCREC/G9nQBGcD2D" + "CLwC2zPNaX2Y9dnyDs2csjN1ibsYttUMnXMgBcnCOkIkVS496Bpc0jQMf35GUgb" + "PSyliwqCoXjEBP/2eyq0VLFKQ0fXGsHWvElT+Y/7RYNTiYVWttFMxN5H/2EGcgn" + "lfNHLpQvXH9u/3YminS9GX30hQ7jFhpHXxkK8gZ1mpHL9K3pfKS3lG6EF9wQ23O" + "qS8m995SG3dp3MzmywxXen/ukXx6bDiEl5VaOvdRUcbhr5Eb3exVDfdWiaJdTYF" + "WfIfJOWx88drB3J9vFwjmuaoNEOjFsoNAMYthYOxXraXaJblvmUKz6tJ3T8/G7x" + "B9QGYNBsOqBolKoKHBtsWCosLdWhEZr9VFFh2AJrOW1fx24CIkHnvfTtwYORvQq" + "Ckuq2bZS1EOdsFkU/X5gwPl6gSUTNhV3IooXkBFL3iBEbfZ6JpQHVVyIuNWjIyN" + "b2liCn9Nn0VHeNMMRLl7uyw4eKlOX2ogom8SLvihYxcJoqlCwtehpLsKsU4iwME" + "PmDteW5GBGf4GbnqPFkpIT5ed1jGhdZt/dpsp+v6QhYH1uX4pPxdkdnuc84/yb9" + "k4SQdKBJ+l3KZkfIxApNWOZqicJfz/eWwS/15hiamRKRuiiUV2zS1V+l8bV7g9O" + "gy5scPBMONxtfFlGEKikZKurFmzboCOGQKRBEUCpsY44IAp443h59pQdVIb0YAS" + "kfp2xKHwYij6ELRNdH5MrlFa3bNTskGO4k5XDR4cl/Sma2SXgBKb5XjTtlNmCQG" + "Gv6lOW7pGXNhs5wfd8K9Ukm6KeLTIlYn1iiKM37YQpa+4JQYljCYhumbqNCkPTZ" + "rNYClh8fQEQ8XuOCDpomMWu58YOTfbZNMDWs/Ou7RfCjX+VNwjPShDK9joMwWKc" + "Jy3QalZbaoWtcyyvXxR2sqhVR9F7Cmasq4=" + ), + 'ssh-xmss@openssh.com': ( + "AAAAFHNzaC14bXNzQG9wZW5zc2guY29tAAAAFVhNU1NfU0hBMi0yNTZfVzE2X0g" + "xMAAAAECqptWnK94d+Sj2xcdTu8gz+75lawZoLSZFqC5IhbYuT/Z3oBZCim6yt+" + "HAmk6MKldl3Fg+74v4sR/SII0I0Jv/" ), } +KEY_TYPES = list(VALID_CONTENT.keys()) + TEST_OPTIONS = ( "no-port-forwarding,no-agent-forwarding,no-X11-forwarding," 'command="echo \'Please login as the user \"ubuntu\" rather than the' @@ -70,13 +287,7 @@ class TestAuthKeyLineParser(test_helpers.CiTestCase): def test_simple_parse(self): # test key line with common 3 fields (keytype, base64, comment) parser = ssh_util.AuthKeyLineParser() - ecdsa_types = [ - 'ecdsa-sha2-nistp256', - 'ecdsa-sha2-nistp384', - 'ecdsa-sha2-nistp521', - ] - - for ktype in ['rsa', 'ecdsa', 'dsa'] + ecdsa_types: + for ktype in KEY_TYPES: content = VALID_CONTENT[ktype] comment = 'user-%s@host' % ktype line = ' '.join((ktype, content, comment,)) @@ -90,7 +301,7 @@ class TestAuthKeyLineParser(test_helpers.CiTestCase): def test_parse_no_comment(self): # test key line with key type and base64 only parser = ssh_util.AuthKeyLineParser() - for ktype in ['rsa', 'ecdsa', 'dsa']: + for ktype in KEY_TYPES: content = VALID_CONTENT[ktype] line = ' '.join((ktype, content,)) key = parser.parse(line) @@ -104,7 +315,7 @@ class TestAuthKeyLineParser(test_helpers.CiTestCase): # test key line with options in it parser = ssh_util.AuthKeyLineParser() options = TEST_OPTIONS - for ktype in ['rsa', 'ecdsa', 'dsa']: + for ktype in KEY_TYPES: content = VALID_CONTENT[ktype] comment = 'user-%s@host' % ktype line = ' '.join((options, ktype, content, comment,)) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index 0c4d728f..c67db436 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -13,6 +13,7 @@ lucasmoura marlluslustosa matthewruffell nishigori +omBratteng onitake smoser sshedi -- cgit v1.2.3 From 07104504ab5b30efd2d1f7a8c36effe18b8e5fe0 Mon Sep 17 00:00:00 2001 From: Paride Legovini Date: Tue, 25 Aug 2020 17:21:18 +0200 Subject: tox: bump the pylint version to 2.6.0 in the default run (#544) Changes: tox: bump the pylint version to 2.6.0 in the default run Fix pylint 2.6.0 W0707 warnings (raise-missing-from) --- cloudinit/analyze/show.py | 2 +- cloudinit/cmd/devel/make_mime.py | 6 ++-- cloudinit/config/cc_disk_setup.py | 36 ++++++++++++++-------- cloudinit/config/cc_growpart.py | 8 ++--- cloudinit/config/cc_power_state_change.py | 12 +++++--- cloudinit/config/cc_rsyslog.py | 6 ++-- cloudinit/config/cc_set_hostname.py | 2 +- cloudinit/config/cc_ubuntu_advantage.py | 2 +- cloudinit/config/schema.py | 2 +- cloudinit/distros/__init__.py | 5 +-- cloudinit/distros/arch.py | 4 +-- cloudinit/distros/parsers/resolv_conf.py | 7 +++-- cloudinit/gpg.py | 5 +-- cloudinit/handlers/jinja_template.py | 3 +- cloudinit/net/__init__.py | 7 +++-- cloudinit/net/cmdline.py | 4 +-- cloudinit/net/dhcp.py | 4 +-- cloudinit/net/network_state.py | 19 +++++++----- cloudinit/sources/DataSourceAzure.py | 2 +- cloudinit/sources/DataSourceEc2.py | 8 +++-- cloudinit/sources/DataSourceIBMCloud.py | 3 +- cloudinit/sources/DataSourceMAAS.py | 3 +- cloudinit/sources/DataSourceOpenNebula.py | 13 +++++--- cloudinit/sources/DataSourceOpenStack.py | 4 +-- cloudinit/sources/DataSourceSmartOS.py | 4 ++- cloudinit/sources/helpers/azure.py | 34 +++++++++++--------- cloudinit/sources/helpers/netlink.py | 2 +- cloudinit/sources/helpers/openstack.py | 34 +++++++++++--------- cloudinit/subp.py | 3 +- cloudinit/url_helper.py | 6 ++-- cloudinit/util.py | 8 ++--- tests/cloud_tests/platforms/azurecloud/instance.py | 7 +++-- tests/cloud_tests/platforms/azurecloud/platform.py | 23 +++++++++----- tests/cloud_tests/platforms/ec2/instance.py | 4 +-- tests/cloud_tests/platforms/ec2/platform.py | 18 ++++++----- tests/cloud_tests/platforms/lxd/instance.py | 3 +- tests/cloud_tests/platforms/platforms.py | 6 ++-- tests/cloud_tests/testcases/__init__.py | 6 ++-- tools/mock-meta.py | 14 +++++---- tox.ini | 2 +- 40 files changed, 205 insertions(+), 136 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py index 0c825b23..01a4d3e5 100644 --- a/cloudinit/analyze/show.py +++ b/cloudinit/analyze/show.py @@ -267,7 +267,7 @@ def gather_timestamps_using_systemd(): except OSError as err: raise RuntimeError('Could not determine container boot ' 'time from /proc/1/cmdline. ({})' - .format(err)) + .format(err)) from err status = CONTAINER_CODE else: status = FAIL_CODE diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index 77e10540..4e6a5778 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -22,8 +22,10 @@ def file_content_type(text): try: filename, content_type = text.split(":", 1) return (open(filename, 'r'), filename, content_type.strip()) - except ValueError: - raise argparse.ArgumentError(text, "Invalid value for %r" % (text)) + except ValueError as e: + raise argparse.ArgumentError( + text, "Invalid value for %r" % (text) + ) from e def get_parser(parser=None): diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index d957cfe3..a7bdc703 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -251,7 +251,9 @@ def enumerate_disk(device, nodeps=False): try: info, _err = subp.subp(lsblk_cmd) except Exception as e: - raise Exception("Failed during disk check for %s\n%s" % (device, e)) + raise Exception( + "Failed during disk check for %s\n%s" % (device, e) + ) from e parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0] @@ -313,7 +315,9 @@ def check_fs(device): try: out, _err = subp.subp(blkid_cmd, rcs=[0, 2]) except Exception as e: - raise Exception("Failed during disk check for %s\n%s" % (device, e)) + raise Exception( + "Failed during disk check for %s\n%s" % (device, e) + ) from e if out: if len(out.splitlines()) == 1: @@ -428,8 +432,8 @@ def get_dyn_func(*args): else: return globals()[func_name] - except KeyError: - raise Exception("No such function %s to call!" % func_name) + except KeyError as e: + raise Exception("No such function %s to call!" % func_name) from e def get_hdd_size(device): @@ -437,7 +441,7 @@ def get_hdd_size(device): size_in_bytes, _ = subp.subp([BLKDEV_CMD, '--getsize64', device]) sector_size, _ = subp.subp([BLKDEV_CMD, '--getss', device]) except Exception as e: - raise Exception("Failed to get %s size\n%s" % (device, e)) + raise Exception("Failed to get %s size\n%s" % (device, e)) from e return int(size_in_bytes) / int(sector_size) @@ -455,8 +459,9 @@ def check_partition_mbr_layout(device, layout): try: out, _err = subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: - raise Exception("Error running partition command on %s\n%s" % ( - device, e)) + raise Exception( + "Error running partition command on %s\n%s" % (device, e) + ) from e found_layout = [] for line in out.splitlines(): @@ -485,8 +490,9 @@ def check_partition_gpt_layout(device, layout): try: out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV) except Exception as e: - raise Exception("Error running partition command on %s\n%s" % ( - device, e)) + raise Exception( + "Error running partition command on %s\n%s" % (device, e) + ) from e out_lines = iter(out.splitlines()) # Skip header. Output looks like: @@ -657,8 +663,10 @@ def purge_disk(device): try: LOG.info("Purging filesystem on /dev/%s", d['name']) subp.subp(wipefs_cmd) - except Exception: - raise Exception("Failed FS purge of /dev/%s" % d['name']) + except Exception as e: + raise Exception( + "Failed FS purge of /dev/%s" % d['name'] + ) from e purge_disk_ptable(device) @@ -700,7 +708,9 @@ def exec_mkpart_mbr(device, layout): try: subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: - raise Exception("Failed to partition device %s\n%s" % (device, e)) + raise Exception( + "Failed to partition device %s\n%s" % (device, e) + ) from e read_parttbl(device) @@ -997,6 +1007,6 @@ def mkfs(fs_cfg): try: subp.subp(fs_cmd, shell=shell) except Exception as e: - raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) + raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) from e # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index c5d93f81..237c3d02 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -148,14 +148,14 @@ class ResizeGrowPart(object): if e.exit_code != 1: util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)", diskdev, partnum) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e return (before, before) try: subp.subp(["growpart", diskdev, partnum]) except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: growpart %s %s", diskdev, partnum) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e return (before, get_size(partdev)) @@ -187,14 +187,14 @@ class ResizeGpart(object): except subp.ProcessExecutionError as e: if e.exit_code != 0: util.logexc(LOG, "Failed: gpart recover %s", diskdev) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e before = get_size(partdev) try: subp.subp(["gpart", "resize", "-i", partnum, diskdev]) except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e # Since growing the FS requires a reboot, make sure we reboot # first when this module has finished. diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index ab953a0d..6fcb8a7d 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -196,10 +196,11 @@ def load_power_state(cfg, distro_name): try: delay = convert_delay(delay, fmt=fmt, scale=scale) - except ValueError: + except ValueError as e: raise TypeError( "power_state[delay] must be 'now' or '+m' (minutes)." - " found '%s'." % delay) + " found '%s'." % delay + ) from e args = command + [delay] if message: @@ -207,9 +208,10 @@ def load_power_state(cfg, distro_name): try: timeout = float(pstate.get('timeout', 30.0)) - except ValueError: - raise ValueError("failed to convert timeout '%s' to float." % - pstate['timeout']) + except ValueError as e: + raise ValueError( + "failed to convert timeout '%s' to float." % pstate['timeout'] + ) from e condition = pstate.get("condition", True) if not isinstance(condition, (str, list, bool)): diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 1354885a..2a2bc931 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -347,8 +347,10 @@ class SyslogRemotesLine(object): if self.port: try: int(self.port) - except ValueError: - raise ValueError("port '%s' is not an integer" % self.port) + except ValueError as e: + raise ValueError( + "port '%s' is not an integer" % self.port + ) from e if not self.addr: raise ValueError("address is required") diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index c13020d8..1d23d80d 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -85,7 +85,7 @@ def handle(name, cfg, cloud, log, _args): except Exception as e: msg = "Failed to set the hostname to %s (%s)" % (fqdn, hostname) util.logexc(log, msg) - raise SetHostnameError("%s: %s" % (msg, e)) + raise SetHostnameError("%s: %s" % (msg, e)) from e write_json(prev_fn, {'hostname': hostname, 'fqdn': fqdn}) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py index 35ded5db..d61dc655 100644 --- a/cloudinit/config/cc_ubuntu_advantage.py +++ b/cloudinit/config/cc_ubuntu_advantage.py @@ -115,7 +115,7 @@ def configure_ua(token=None, enable=None): msg = 'Failure attaching Ubuntu Advantage:\n{error}'.format( error=str(e)) util.logexc(LOG, msg) - raise RuntimeError(msg) + raise RuntimeError(msg) from e enable_errors = [] for service in enable: try: diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 2d8c7577..8a966aee 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -210,7 +210,7 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): error = SchemaValidationError(errors) if annotate: print(annotated_cloudconfig_file({}, content, error.schema_errors)) - raise error + raise error from e try: validate_cloudconfig_schema( cloudconfig, schema, strict=True) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index effb4276..2537608f 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -607,10 +607,11 @@ class Distro(metaclass=abc.ABCMeta): lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name]) try: cmd = next(tool for tool in lock_tools if subp.which(tool[0])) - except StopIteration: + except StopIteration as e: raise RuntimeError(( "Unable to lock user account '%s'. No tools available. " - " Tried: %s.") % (name, [c[0] for c in lock_tools])) + " Tried: %s.") % (name, [c[0] for c in lock_tools]) + ) from e try: subp.subp(cmd) except Exception as e: diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 038aa9ac..967be168 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -61,9 +61,9 @@ class Distro(distros.Distro): def _write_network_config(self, netconfig): try: return self._supported_write_network_config(netconfig) - except RendererNotFoundError: + except RendererNotFoundError as e: # Fall back to old _write_network - raise NotImplementedError + raise NotImplementedError from e def _write_network(self, settings): entries = net_util.translate_network(settings) diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py index 299d54b5..62929d03 100644 --- a/cloudinit/distros/parsers/resolv_conf.py +++ b/cloudinit/distros/parsers/resolv_conf.py @@ -150,9 +150,10 @@ class ResolvConf(object): tail = '' try: (cfg_opt, cfg_values) = head.split(None, 1) - except (IndexError, ValueError): - raise IOError("Incorrectly formatted resolv.conf line %s" - % (i + 1)) + except (IndexError, ValueError) as e: + raise IOError( + "Incorrectly formatted resolv.conf line %s" % (i + 1) + ) from e if cfg_opt not in ['nameserver', 'domain', 'search', 'sortlist', 'options']: raise IOError("Unexpected resolv.conf option %s" % (cfg_opt)) diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py index 72b5ac59..be0ca0ea 100644 --- a/cloudinit/gpg.py +++ b/cloudinit/gpg.py @@ -63,10 +63,11 @@ def recv_key(key, keyserver, retries=(1, 1)): "Import failed with exit code %d, will try again in %ss", error.exit_code, naplen) time.sleep(naplen) - except StopIteration: + except StopIteration as e: raise ValueError( ("Failed to import key '%s' from keyserver '%s' " - "after %d tries: %s") % (key, keyserver, trynum, error)) + "after %d tries: %s") % (key, keyserver, trynum, error) + ) from e def delete_key(key): diff --git a/cloudinit/handlers/jinja_template.py b/cloudinit/handlers/jinja_template.py index ce3accf6..aadfbf86 100644 --- a/cloudinit/handlers/jinja_template.py +++ b/cloudinit/handlers/jinja_template.py @@ -83,7 +83,8 @@ def render_jinja_payload_from_file( if e.errno == EACCES: raise RuntimeError( 'Cannot render jinja template vars. No read permission on' - " '%s'. Try sudo" % instance_data_file) + " '%s'. Try sudo" % instance_data_file + ) from e rendered_payload = render_jinja_payload( payload, payload_fn, instance_data, debug) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 322af77b..e233149a 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -506,7 +506,9 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): try: _rename_interfaces(extract_physdevs(netcfg)) except RuntimeError as e: - raise RuntimeError('Failed to apply network config names: %s' % e) + raise RuntimeError( + 'Failed to apply network config names: %s' % e + ) from e def interface_has_own_mac(ifname, strict=False): @@ -965,7 +967,8 @@ class EphemeralIPv4Network(object): self.prefix = mask_to_net_prefix(prefix_or_mask) except ValueError as e: raise ValueError( - 'Cannot setup network: {0}'.format(e)) + 'Cannot setup network: {0}'.format(e) + ) from e self.connectivity_url = connectivity_url self.interface = interface diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 7ca7262b..cc8dc17b 100755 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -112,8 +112,8 @@ def _klibc_to_config_entry(content, mac_addrs=None): data = util.load_shell_content(content) try: name = data['DEVICE'] if 'DEVICE' in data else data['DEVICE6'] - except KeyError: - raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data") + except KeyError as e: + raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data") from e # ipconfig on precise does not write PROTO # IPv6 config gives us IPV6PROTO, not PROTO. diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 9230cf7a..4394c68b 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -82,8 +82,8 @@ class EphemeralDHCPv4(object): try: leases = maybe_perform_dhcp_discovery( self.iface, self.dhcp_log_func) - except InvalidDHCPLeaseFileError: - raise NoDHCPLeaseError() + except InvalidDHCPLeaseFileError as e: + raise NoDHCPLeaseError() from e if not leases: raise NoDHCPLeaseError() self.lease = leases[-1] diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 7bfe8be0..b2f7d31e 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -297,9 +297,10 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): command_type = command['type'] try: handler = self.command_handlers[command_type] - except KeyError: - raise RuntimeError("No handler found for" - " command '%s'" % command_type) + except KeyError as e: + raise RuntimeError( + "No handler found for command '%s'" % command_type + ) from e try: handler(self, command) except InvalidCommand: @@ -316,9 +317,10 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): continue try: handler = self.command_handlers[command_type] - except KeyError: - raise RuntimeError("No handler found for" - " command '%s'" % command_type) + except KeyError as e: + raise RuntimeError( + "No handler found for command '%s'" % command_type + ) from e try: handler(self, command) self._v2_common(command) @@ -914,9 +916,10 @@ def _normalize_route(route): if metric: try: normal_route['metric'] = int(metric) - except ValueError: + except ValueError as e: raise TypeError( - 'Route config metric {} is not an integer'.format(metric)) + 'Route config metric {} is not an integer'.format(metric) + ) from e return normal_route diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 86cc7c28..f3c6452b 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -1147,7 +1147,7 @@ def read_azure_ovf(contents): except Exception as e: error_str = "Invalid ovf-env.xml: %s" % e report_diagnostic_event(error_str) - raise BrokenAzureDataSource(error_str) + raise BrokenAzureDataSource(error_str) from e results = find_child(dom.documentElement, lambda n: n.localName == "ProvisioningSection") diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 355b4e2f..1d09c12a 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -617,9 +617,11 @@ def parse_strict_mode(cfgval): if sleep: try: sleep = int(sleep) - except ValueError: - raise ValueError("Invalid sleep '%s' in strict_id setting '%s': " - "not an integer" % (sleep, cfgval)) + except ValueError as e: + raise ValueError( + "Invalid sleep '%s' in strict_id setting '%s': not an integer" + % (sleep, cfgval) + ) from e else: sleep = None diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py index d2aa9a58..8d196185 100644 --- a/cloudinit/sources/DataSourceIBMCloud.py +++ b/cloudinit/sources/DataSourceIBMCloud.py @@ -303,7 +303,8 @@ def read_md(): except sources.BrokenMetadata as e: raise RuntimeError( "Failed reading IBM config disk (platform=%s path=%s): %s" % - (platform, path, e)) + (platform, path, e) + ) from e ret.update(results) return ret diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index c80f70c2..9156925f 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -226,7 +226,8 @@ def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None, except url_helper.UrlError as e: if e.code == 404 and not optional: raise MAASSeedDirMalformed( - "Missing required %s: %s" % (path, e)) + "Missing required %s: %s" % (path, e) + ) from e elif e.code != 404: raise e diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 12b1f94f..45481938 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -399,18 +399,23 @@ def read_context_disk_dir(source_dir, distro, asuser=None): if asuser is not None: try: pwd.getpwnam(asuser) - except KeyError: + except KeyError as e: raise BrokenContextDiskDir( "configured user '{user}' does not exist".format( - user=asuser)) + user=asuser) + ) from e try: path = os.path.join(source_dir, 'context.sh') content = util.load_file(path) context = parse_shell_config(content, asuser=asuser) except subp.ProcessExecutionError as e: - raise BrokenContextDiskDir("Error processing context.sh: %s" % (e)) + raise BrokenContextDiskDir( + "Error processing context.sh: %s" % (e) + ) from e except IOError as e: - raise NonContextDiskDir("Error reading context.sh: %s" % (e)) + raise NonContextDiskDir( + "Error reading context.sh: %s" % (e) + ) from e else: raise NonContextDiskDir("Missing context.sh") diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index bf539091..d4b43f44 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -194,10 +194,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'timeout': url_params.timeout_seconds}) except openstack.NonReadable as e: raise sources.InvalidMetaDataException(str(e)) - except (openstack.BrokenMetadata, IOError): + except (openstack.BrokenMetadata, IOError) as e: msg = 'Broken metadata address {addr}'.format( addr=self.metadata_address) - raise sources.InvalidMetaDataException(msg) + raise sources.InvalidMetaDataException(msg) from e return result diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 843b3a2a..f1f903bc 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -413,7 +413,9 @@ class JoyentMetadataClient(object): response.append(byte) except OSError as exc: if exc.errno == errno.EAGAIN: - raise JoyentMetadataTimeoutException(msg % as_ascii()) + raise JoyentMetadataTimeoutException( + msg % as_ascii() + ) from exc raise def _write(self, msg): diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 6156c75b..b968a96f 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -98,8 +98,10 @@ def get_boot_telemetry(): LOG.debug("Collecting boot telemetry") try: kernel_start = float(time.time()) - float(util.uptime()) - except ValueError: - raise RuntimeError("Failed to determine kernel start timestamp") + except ValueError as e: + raise RuntimeError( + "Failed to determine kernel start timestamp" + ) from e try: out, _ = subp.subp(['/bin/systemctl', @@ -116,12 +118,13 @@ def get_boot_telemetry(): user_start = kernel_start + (float(tsm) / 1000000) except subp.ProcessExecutionError as e: - raise RuntimeError("Failed to get UserspaceTimestampMonotonic: %s" - % e) + raise RuntimeError( + "Failed to get UserspaceTimestampMonotonic: %s" % e + ) from e except ValueError as e: - raise RuntimeError("Failed to parse " - "UserspaceTimestampMonotonic from systemd: %s" - % e) + raise RuntimeError( + "Failed to parse UserspaceTimestampMonotonic from systemd: %s" % e + ) from e try: out, _ = subp.subp(['/bin/systemctl', 'show', @@ -137,12 +140,14 @@ def get_boot_telemetry(): cloudinit_activation = kernel_start + (float(tsm) / 1000000) except subp.ProcessExecutionError as e: - raise RuntimeError("Failed to get InactiveExitTimestampMonotonic: %s" - % e) + raise RuntimeError( + "Failed to get InactiveExitTimestampMonotonic: %s" % e + ) from e except ValueError as e: - raise RuntimeError("Failed to parse " - "InactiveExitTimestampMonotonic from systemd: %s" - % e) + raise RuntimeError( + "Failed to parse InactiveExitTimestampMonotonic from systemd: %s" + % e + ) from e evt = events.ReportingEvent( BOOT_EVENT_TYPE, 'boot-telemetry', @@ -642,9 +647,10 @@ class WALinuxAgentShim: try: name = os.path.basename(hook_file).replace('.json', '') dhcp_options[name] = json.loads(util.load_file((hook_file))) - except ValueError: + except ValueError as e: raise ValueError( - '{_file} is not valid JSON data'.format(_file=hook_file)) + '{_file} is not valid JSON data'.format(_file=hook_file) + ) from e return dhcp_options @staticmethod diff --git a/cloudinit/sources/helpers/netlink.py b/cloudinit/sources/helpers/netlink.py index a74a3588..c2ad587b 100644 --- a/cloudinit/sources/helpers/netlink.py +++ b/cloudinit/sources/helpers/netlink.py @@ -74,7 +74,7 @@ def create_bound_netlink_socket(): netlink_socket.setblocking(0) except socket.error as e: msg = "Exception during netlink socket create: %s" % e - raise NetlinkCreateSocketError(msg) + raise NetlinkCreateSocketError(msg) from e LOG.debug("Created netlink socket") return netlink_socket diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 1050efb0..65e020c5 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -280,8 +280,9 @@ class BaseReader(metaclass=abc.ABCMeta): try: data = translator(data) except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to process path %s: %s" % (path, e) + ) from e if found: results[name] = data @@ -291,8 +292,9 @@ class BaseReader(metaclass=abc.ABCMeta): try: metadata['random_seed'] = base64.b64decode(random_seed) except (ValueError, TypeError) as e: - raise BrokenMetadata("Badly formatted metadata" - " random_seed entry: %s" % e) + raise BrokenMetadata( + "Badly formatted metadata random_seed entry: %s" % e + ) from e # load any files that were provided files = {} @@ -304,8 +306,9 @@ class BaseReader(metaclass=abc.ABCMeta): try: files[path] = self._read_content_path(item) except Exception as e: - raise BrokenMetadata("Failed to read provided " - "file %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to read provided file %s: %s" % (path, e) + ) from e results['files'] = files # The 'network_config' item in metadata is a content pointer @@ -317,8 +320,9 @@ class BaseReader(metaclass=abc.ABCMeta): content = self._read_content_path(net_item, decode=True) results['network_config'] = content except IOError as e: - raise BrokenMetadata("Failed to read network" - " configuration: %s" % (e)) + raise BrokenMetadata( + "Failed to read network configuration: %s" % (e) + ) from e # To openstack, user can specify meta ('nova boot --meta=key=value') # and those will appear under metadata['meta']. @@ -370,8 +374,9 @@ class ConfigDriveReader(BaseReader): try: return util.load_json(self._path_read(path)) except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to process path %s: %s" % (path, e) + ) from e def read_v1(self): """Reads a version 1 formatted location. @@ -395,16 +400,17 @@ class ConfigDriveReader(BaseReader): path = found[name] try: contents = self._path_read(path) - except IOError: - raise BrokenMetadata("Failed to read: %s" % path) + except IOError as e: + raise BrokenMetadata("Failed to read: %s" % path) from e try: # Disable not-callable pylint check; pylint isn't able to # determine that every member of FILES_V1 has a callable in # the appropriate position md[key] = translator(contents) # pylint: disable=E1102 except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to process path %s: %s" % (path, e) + ) from e else: md[key] = copy.deepcopy(default) diff --git a/cloudinit/subp.py b/cloudinit/subp.py index 804ef3ca..3e4efa42 100644 --- a/cloudinit/subp.py +++ b/cloudinit/subp.py @@ -262,7 +262,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True, raise ProcessExecutionError( cmd=args, reason=e, errno=e.errno, stdout="-" if decode else b"-", - stderr="-" if decode else b"-") + stderr="-" if decode else b"-" + ) from e finally: if devnull_fp: devnull_fp.close() diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index f3c0cf9c..caa88435 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -95,7 +95,7 @@ def read_file_or_url(url, **kwargs): code = e.errno if e.errno == ENOENT: code = NOT_FOUND - raise UrlError(cause=e, code=code, headers=None, url=url) + raise UrlError(cause=e, code=code, headers=None, url=url) from e return FileResponse(file_path, contents=contents) else: return readurl(url, **kwargs) @@ -575,8 +575,8 @@ def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret, timestamp=None): try: import oauthlib.oauth1 as oauth1 - except ImportError: - raise NotImplementedError('oauth support is not available') + except ImportError as e: + raise NotImplementedError('oauth support is not available') from e if timestamp: timestamp = str(timestamp) diff --git a/cloudinit/util.py b/cloudinit/util.py index dd263803..cf9e349f 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -359,7 +359,7 @@ def decomp_gzip(data, quiet=True, decode=True): if quiet: return data else: - raise DecompressionError(str(e)) + raise DecompressionError(str(e)) from e def extract_usergroup(ug_pair): @@ -1363,7 +1363,7 @@ def chownbyname(fname, user=None, group=None): if group: gid = grp.getgrnam(group).gr_gid except KeyError as e: - raise OSError("Unknown user or group: %s" % (e)) + raise OSError("Unknown user or group: %s" % (e)) from e chownbyid(fname, uid, gid) @@ -2387,8 +2387,8 @@ def human2bytes(size): try: num = float(num) - except ValueError: - raise ValueError("'%s' is not valid input." % size_in) + except ValueError as e: + raise ValueError("'%s' is not valid input." % size_in) from e if num < 0: raise ValueError("'%s': cannot be negative" % size_in) diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py index a136cf0d..eedbaae8 100644 --- a/tests/cloud_tests/platforms/azurecloud/instance.py +++ b/tests/cloud_tests/platforms/azurecloud/instance.py @@ -134,9 +134,10 @@ class AzureCloudInstance(Instance): self.vm_name, vm_params) LOG.debug('creating instance %s from image_id=%s', self.vm_name, self.image_id) - except CloudError: - raise RuntimeError('failed creating instance:\n{}'.format( - traceback.format_exc())) + except CloudError as e: + raise RuntimeError( + 'failed creating instance:\n{}'.format(traceback.format_exc()) + ) from e if wait: self.instance.wait() diff --git a/tests/cloud_tests/platforms/azurecloud/platform.py b/tests/cloud_tests/platforms/azurecloud/platform.py index cb62a74b..a664f612 100644 --- a/tests/cloud_tests/platforms/azurecloud/platform.py +++ b/tests/cloud_tests/platforms/azurecloud/platform.py @@ -59,9 +59,12 @@ class AzureCloudPlatform(Platform): self.vnet = self._create_vnet() self.subnet = self._create_subnet() self.nic = self._create_nic() - except CloudError: - raise RuntimeError('failed creating a resource:\n{}'.format( - traceback.format_exc())) + except CloudError as e: + raise RuntimeError( + 'failed creating a resource:\n{}'.format( + traceback.format_exc() + ) + ) from e def create_instance(self, properties, config, features, image_id, user_data=None): @@ -105,8 +108,10 @@ class AzureCloudPlatform(Platform): if image_id.find('__') > 0: image_id = image_id.split('__')[1] LOG.debug('image_id shortened to %s', image_id) - except KeyError: - raise RuntimeError('no images found for %s' % img_conf['release']) + except KeyError as e: + raise RuntimeError( + 'no images found for %s' % img_conf['release'] + ) from e return AzureCloudImage(self, img_conf, image_id) @@ -140,9 +145,11 @@ class AzureCloudPlatform(Platform): secret=azure_creds['clientSecret'], tenant=azure_creds['tenantId']) return credentials, subscription_id - except KeyError: - raise RuntimeError('Please configure Azure service principal' - ' credentials in %s' % cred_file) + except KeyError as e: + raise RuntimeError( + 'Please configure Azure service principal' + ' credentials in %s' % cred_file + ) from e def _create_resource_group(self): """Create resource group""" diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py index ab6037b1..d2e84047 100644 --- a/tests/cloud_tests/platforms/ec2/instance.py +++ b/tests/cloud_tests/platforms/ec2/instance.py @@ -49,11 +49,11 @@ class EC2Instance(Instance): # OutputBytes comes from platform._decode_console_output_as_bytes response = self.instance.console_output() return response['OutputBytes'] - except KeyError: + except KeyError as e: if 'Output' in response: msg = ("'OutputBytes' did not exist in console_output() but " "'Output' did: %s..." % response['Output'][0:128]) - raise util.PlatformError('console_log', msg) + raise util.PlatformError('console_log', msg) from e return ('No Console Output [%s]' % self.instance).encode() def destroy(self): diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py index 7a3d0fe0..b61a2ffb 100644 --- a/tests/cloud_tests/platforms/ec2/platform.py +++ b/tests/cloud_tests/platforms/ec2/platform.py @@ -35,12 +35,14 @@ class EC2Platform(Platform): self.ec2_resource = b3session.resource('ec2') self.ec2_region = b3session.region_name self.key_name = self._upload_public_key(config) - except botocore.exceptions.NoRegionError: + except botocore.exceptions.NoRegionError as e: raise RuntimeError( - 'Please configure default region in $HOME/.aws/config') - except botocore.exceptions.NoCredentialsError: + 'Please configure default region in $HOME/.aws/config' + ) from e + except botocore.exceptions.NoCredentialsError as e: raise RuntimeError( - 'Please configure ec2 credentials in $HOME/.aws/credentials') + 'Please configure ec2 credentials in $HOME/.aws/credentials' + ) from e self.vpc = self._create_vpc() self.internet_gateway = self._create_internet_gateway() @@ -125,8 +127,10 @@ class EC2Platform(Platform): try: image_ami = image['id'] - except KeyError: - raise RuntimeError('No images found for %s!' % img_conf['release']) + except KeyError as e: + raise RuntimeError( + 'No images found for %s!' % img_conf['release'] + ) from e LOG.debug('found image: %s', image_ami) image = EC2Image(self, img_conf, image_ami) @@ -195,7 +199,7 @@ class EC2Platform(Platform): CidrBlock=self.ipv4_cidr, AmazonProvidedIpv6CidrBlock=True) except botocore.exceptions.ClientError as e: - raise RuntimeError(e) + raise RuntimeError(e) from e vpc.wait_until_available() self._tag_resource(vpc) diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py index b27b9848..2b973a08 100644 --- a/tests/cloud_tests/platforms/lxd/instance.py +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -175,7 +175,8 @@ class LXDInstance(Instance): raise PlatformError( "console log", "Console log failed [%d]: stdout=%s stderr=%s" % ( - e.exit_code, e.stdout, e.stderr)) + e.exit_code, e.stdout, e.stderr) + ) from e def reboot(self, wait=True): """Reboot instance.""" diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py index 58f65e52..ac3b6563 100644 --- a/tests/cloud_tests/platforms/platforms.py +++ b/tests/cloud_tests/platforms/platforms.py @@ -74,8 +74,10 @@ class Platform(object): try: return tmirror.json_entries[0] - except IndexError: - raise RuntimeError('no images found with filter: %s' % img_filter) + except IndexError as e: + raise RuntimeError( + 'no images found with filter: %s' % img_filter + ) from e class FilterMirror(mirrors.BasicMirrorWriter): diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py index e8c371ca..bb9785d3 100644 --- a/tests/cloud_tests/testcases/__init__.py +++ b/tests/cloud_tests/testcases/__init__.py @@ -21,8 +21,10 @@ def discover_test(test_name): config.name_sanitize(test_name)) try: testmod = importlib.import_module(testmod_name) - except NameError: - raise ValueError('no test verifier found at: {}'.format(testmod_name)) + except NameError as e: + raise ValueError( + 'no test verifier found at: {}'.format(testmod_name) + ) from e found = [mod for name, mod in inspect.getmembers(testmod) if (inspect.isclass(mod) diff --git a/tools/mock-meta.py b/tools/mock-meta.py index a58e0260..9dd067b9 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -258,12 +258,14 @@ class MetaDataHandler(object): try: key_id = int(mybe_key) key_name = key_ids[key_id] - except ValueError: - raise WebException(hclient.BAD_REQUEST, - "%s: not an integer" % mybe_key) - except IndexError: - raise WebException(hclient.NOT_FOUND, - "Unknown key id %r" % mybe_key) + except ValueError as e: + raise WebException( + hclient.BAD_REQUEST, "%s: not an integer" % mybe_key + ) from e + except IndexError as e: + raise WebException( + hclient.NOT_FOUND, "Unknown key id %r" % mybe_key + ) from e # Extract the possible sub-params result = traverse(nparams[1:], { "openssh-key": "\n".join(avail_keys[key_name]), diff --git a/tox.ini b/tox.ini index f619dbf5..a92c63e0 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ setenv = basepython = python3 deps = # requirements - pylint==2.5.3 + pylint==2.6.0 # test-requirements because unit tests are now present in cloudinit tree -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt -- cgit v1.2.3