From 3c551f6ebc12f7729a2755c89b19b9000e27cc88 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 8 Jun 2020 12:49:12 -0400 Subject: Move subp into its own module. (#416) This was painful, but it finishes a TODO from cloudinit/subp.py. It moves the following from util to subp: ProcessExecutionError subp which target_path I moved subp_blob_in_tempfile into cc_chef, which is its only caller. That saved us from having to deal with it using write_file and temp_utils from subp (which does not import any cloudinit things now). It is arguable that 'target_path' could be moved to a 'path_utils' or something, but in order to use it from subp and also from utils, we had to get it out of utils. --- cloudinit/net/dhcp.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'cloudinit/net/dhcp.py') diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 19d0199c..d03baeab 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -17,6 +17,7 @@ from cloudinit.net import ( has_url_connectivity) from cloudinit.net.network_state import mask_and_ipv4_to_bcast_addr as bcip from cloudinit import temp_utils +from cloudinit import subp from cloudinit import util LOG = logging.getLogger(__name__) @@ -150,7 +151,7 @@ def maybe_perform_dhcp_discovery(nic=None): LOG.debug( 'Skip dhcp_discovery: nic %s not found in get_devicelist.', nic) return [] - dhclient_path = util.which('dhclient') + dhclient_path = subp.which('dhclient') if not dhclient_path: LOG.debug('Skip dhclient configuration: No dhclient command found.') return [] @@ -219,10 +220,10 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir): # Generally dhclient relies on dhclient-script PREINIT action to bring the # link up before attempting discovery. Since we are using -sf /bin/true, # we need to do that "link up" ourselves first. - util.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True) + subp.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True) cmd = [sandbox_dhclient_cmd, '-1', '-v', '-lf', lease_file, '-pf', pid_file, interface, '-sf', '/bin/true'] - util.subp(cmd, capture=True) + subp.subp(cmd, capture=True) # Wait for pid file and lease file to appear, and for the process # named by the pid file to daemonize (have pid 1 as its parent). If we -- cgit v1.2.3 From d083a0315faf67c00acdecdc3f95c700edf6ba06 Mon Sep 17 00:00:00 2001 From: Moustafa Moustafa Date: Fri, 19 Jun 2020 06:38:05 -0700 Subject: printing the error stream of the dhclient process before killing it (#369) This introduces a way to log the dhclient error stream, and uses it for the Azure datasource (where we have a specific requirement for this data to be logged). --- cloudinit/net/dhcp.py | 33 +++++++++++++------- cloudinit/net/tests/test_dhcp.py | 38 +++++++++++++++++++++++ cloudinit/sources/DataSourceAzure.py | 6 ++-- cloudinit/sources/helpers/azure.py | 8 ++++- tests/unittests/test_datasource/test_ec2.py | 2 +- tests/unittests/test_datasource/test_openstack.py | 2 +- 6 files changed, 73 insertions(+), 16 deletions(-) (limited to 'cloudinit/net/dhcp.py') diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index d03baeab..9e004688 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -40,10 +40,11 @@ class NoDHCPLeaseError(Exception): class EphemeralDHCPv4(object): - def __init__(self, iface=None, connectivity_url=None): + def __init__(self, iface=None, connectivity_url=None, dhcp_log_func=None): self.iface = iface self._ephipv4 = None self.lease = None + self.dhcp_log_func = dhcp_log_func self.connectivity_url = connectivity_url def __enter__(self): @@ -81,7 +82,8 @@ class EphemeralDHCPv4(object): if self.lease: return self.lease try: - leases = maybe_perform_dhcp_discovery(self.iface) + leases = maybe_perform_dhcp_discovery( + self.iface, self.dhcp_log_func) except InvalidDHCPLeaseFileError: raise NoDHCPLeaseError() if not leases: @@ -131,13 +133,15 @@ class EphemeralDHCPv4(object): result[internal_mapping] = self.lease.get(different_names) -def maybe_perform_dhcp_discovery(nic=None): +def maybe_perform_dhcp_discovery(nic=None, dhcp_log_func=None): """Perform dhcp discovery if nic valid and dhclient command exists. If the nic is invalid or undiscoverable or dhclient command is not found, skip dhcp_discovery and return an empty dict. @param nic: Name of the network interface we want to run dhclient on. + @param dhcp_log_func: A callable accepting the dhclient output and error + streams. @return: A list of dicts representing dhcp options for each lease obtained from the dhclient discovery if run, otherwise an empty list is returned. @@ -159,7 +163,7 @@ def maybe_perform_dhcp_discovery(nic=None): prefix='cloud-init-dhcp-', needs_exe=True) as tdir: # Use /var/tmp because /run/cloud-init/tmp is mounted noexec - return dhcp_discovery(dhclient_path, nic, tdir) + return dhcp_discovery(dhclient_path, nic, tdir, dhcp_log_func) def parse_dhcp_lease_file(lease_file): @@ -193,13 +197,15 @@ def parse_dhcp_lease_file(lease_file): return dhcp_leases -def dhcp_discovery(dhclient_cmd_path, interface, cleandir): +def dhcp_discovery(dhclient_cmd_path, interface, cleandir, dhcp_log_func=None): """Run dhclient on the interface without scripts or filesystem artifacts. @param dhclient_cmd_path: Full path to the dhclient used. @param interface: Name of the network inteface on which to dhclient. @param cleandir: The directory from which to run dhclient as well as store dhcp leases. + @param dhcp_log_func: A callable accepting the dhclient output and error + streams. @return: A list of dicts of representing the dhcp leases parsed from the dhcp.leases file or empty list. @@ -223,7 +229,7 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir): subp.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True) cmd = [sandbox_dhclient_cmd, '-1', '-v', '-lf', lease_file, '-pf', pid_file, interface, '-sf', '/bin/true'] - subp.subp(cmd, capture=True) + out, err = subp.subp(cmd, capture=True) # Wait for pid file and lease file to appear, and for the process # named by the pid file to daemonize (have pid 1 as its parent). If we @@ -240,6 +246,7 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir): return [] ppid = 'unknown' + daemonized = False for _ in range(0, 1000): pid_content = util.load_file(pid_file).strip() try: @@ -251,13 +258,17 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir): if ppid == 1: LOG.debug('killing dhclient with pid=%s', pid) os.kill(pid, signal.SIGKILL) - return parse_dhcp_lease_file(lease_file) + daemonized = True + break time.sleep(0.01) - LOG.error( - 'dhclient(pid=%s, parentpid=%s) failed to daemonize after %s seconds', - pid_content, ppid, 0.01 * 1000 - ) + if not daemonized: + LOG.error( + 'dhclient(pid=%s, parentpid=%s) failed to daemonize after %s ' + 'seconds', pid_content, ppid, 0.01 * 1000 + ) + if dhcp_log_func is not None: + dhcp_log_func(out, err) return parse_dhcp_lease_file(lease_file) diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index d4881592..7987f9d1 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -309,6 +309,7 @@ class TestDHCPDiscoveryClean(CiTestCase): Lease processing still occurs and no proc kill is attempted. """ + m_subp.return_value = ('', '') tmpdir = self.tmp_dir() dhclient_script = os.path.join(tmpdir, 'dhclient.orig') script_content = '#!/bin/bash\necho fake-dhclient' @@ -344,6 +345,7 @@ class TestDHCPDiscoveryClean(CiTestCase): m_kill, m_getppid): """dhcp_discovery waits for the presence of pidfile and dhcp.leases.""" + m_subp.return_value = ('', '') tmpdir = self.tmp_dir() dhclient_script = os.path.join(tmpdir, 'dhclient.orig') script_content = '#!/bin/bash\necho fake-dhclient' @@ -370,6 +372,7 @@ class TestDHCPDiscoveryClean(CiTestCase): It also returns the parsed dhcp.leases file generated in the sandbox. """ + m_subp.return_value = ('', '') tmpdir = self.tmp_dir() dhclient_script = os.path.join(tmpdir, 'dhclient.orig') script_content = '#!/bin/bash\necho fake-dhclient' @@ -406,6 +409,41 @@ class TestDHCPDiscoveryClean(CiTestCase): 'eth9', '-sf', '/bin/true'], capture=True)]) m_kill.assert_has_calls([mock.call(my_pid, signal.SIGKILL)]) + @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid') + @mock.patch('cloudinit.net.dhcp.os.kill') + @mock.patch('cloudinit.net.dhcp.subp.subp') + def test_dhcp_output_error_stream(self, m_subp, m_kill, m_getppid): + """"dhcp_log_func is called with the output and error streams of + dhclinet when the callable is passed.""" + dhclient_err = 'FAKE DHCLIENT ERROR' + dhclient_out = 'FAKE DHCLIENT OUT' + m_subp.return_value = (dhclient_out, dhclient_err) + tmpdir = self.tmp_dir() + dhclient_script = os.path.join(tmpdir, 'dhclient.orig') + script_content = '#!/bin/bash\necho fake-dhclient' + write_file(dhclient_script, script_content, mode=0o755) + lease_content = dedent(""" + lease { + interface "eth9"; + fixed-address 192.168.2.74; + option subnet-mask 255.255.255.0; + option routers 192.168.2.1; + } + """) + lease_file = os.path.join(tmpdir, 'dhcp.leases') + write_file(lease_file, lease_content) + pid_file = os.path.join(tmpdir, 'dhclient.pid') + my_pid = 1 + write_file(pid_file, "%d\n" % my_pid) + m_getppid.return_value = 1 # Indicate that dhclient has daemonized + + def dhcp_log_func(out, err): + self.assertEqual(out, dhclient_out) + self.assertEqual(err, dhclient_err) + + dhcp_discovery( + dhclient_script, 'eth9', tmpdir, dhcp_log_func=dhcp_log_func) + class TestSystemdParseLeases(CiTestCase): diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 6d569057..41431a7a 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -35,7 +35,8 @@ from cloudinit.sources.helpers.azure import ( get_system_info, report_diagnostic_event, EphemeralDHCPv4WithReporting, - is_byte_swapped) + is_byte_swapped, + dhcp_log_cb) LOG = logging.getLogger(__name__) @@ -636,7 +637,8 @@ class DataSourceAzure(sources.DataSource): name="obtain-dhcp-lease", description="obtain dhcp lease", parent=azure_ds_reporter): - self._ephemeral_dhcp_ctx = EphemeralDHCPv4() + self._ephemeral_dhcp_ctx = EphemeralDHCPv4( + dhcp_log_func=dhcp_log_cb) lease = self._ephemeral_dhcp_ctx.obtain_lease() if vnet_switched: diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 7bace8ca..7afa7ed8 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -626,10 +626,16 @@ def get_metadata_from_fabric(fallback_lease_file=None, dhcp_opts=None, shim.clean_up() +def dhcp_log_cb(out, err): + report_diagnostic_event("dhclient output stream: %s" % out) + report_diagnostic_event("dhclient error stream: %s" % err) + + class EphemeralDHCPv4WithReporting(object): def __init__(self, reporter, nic=None): self.reporter = reporter - self.ephemeralDHCPv4 = EphemeralDHCPv4(iface=nic) + self.ephemeralDHCPv4 = EphemeralDHCPv4( + iface=nic, dhcp_log_func=dhcp_log_cb) def __enter__(self): with events.ReportEventStack( diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index 80e69c03..a93f2195 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -744,7 +744,7 @@ class TestEc2(test_helpers.HttprettyTestCase): ret = ds.get_data() self.assertTrue(ret) - m_dhcp.assert_called_once_with('eth9') + m_dhcp.assert_called_once_with('eth9', None) m_net.assert_called_once_with( broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9', prefix_or_mask='255.255.255.0', router='192.168.2.1', diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index b534457c..3cfba74d 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -279,7 +279,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): self.assertEqual(2, len(ds_os_local.files)) self.assertEqual(VENDOR_DATA, ds_os_local.vendordata_pure) self.assertIsNone(ds_os_local.vendordata_raw) - m_dhcp.assert_called_with('eth9') + m_dhcp.assert_called_with('eth9', None) def test_bad_datasource_meta(self): os_files = copy.deepcopy(OS_FILES) -- cgit v1.2.3 From 4fe576516d65feda17ba78e9265a8e494a195e7b Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 15 Jul 2020 10:26:12 -0400 Subject: cloudinit: remove global disable of pylint W0107 and fix errors (#489) * cloudinit: remove global disable of pylint W0107 and fix errors This includes removing a test class which contained no tests but wasn't detected as empty because of an errant pass statement. * .pylintrc: update disable comment to match arguments --- .pylintrc | 4 +--- cloudinit/config/cc_set_hostname.py | 1 - cloudinit/distros/networking.py | 3 --- cloudinit/distros/ubuntu.py | 2 -- cloudinit/net/cmdline.py | 2 -- cloudinit/net/dhcp.py | 2 -- cloudinit/reporting/handlers.py | 1 - cloudinit/sources/DataSourceAzure.py | 1 - cloudinit/sources/__init__.py | 2 -- cloudinit/sources/helpers/netlink.py | 1 - cloudinit/sources/helpers/vmware/imc/config_file.py | 1 - cloudinit/sources/helpers/vmware/imc/config_namespace.py | 1 - cloudinit/sources/helpers/vmware/imc/config_source.py | 1 - cloudinit/stages.py | 1 - tests/cloud_tests/platforms/azurecloud/instance.py | 2 -- tests/cloud_tests/platforms/images.py | 1 - tests/cloud_tests/platforms/snapshots.py | 1 - tests/cloud_tests/testcases/base.py | 1 - tests/unittests/test_datasource/test_ibmcloud.py | 7 ------- tests/unittests/test_reporting.py | 1 - 20 files changed, 1 insertion(+), 35 deletions(-) (limited to 'cloudinit/net/dhcp.py') diff --git a/.pylintrc b/.pylintrc index f04603b9..94a81d0e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -7,7 +7,6 @@ jobs=4 [MESSAGES CONTROL] # Errors and warings with some filtered: -# W0107(unnecessary-pass) # W0201(attribute-defined-outside-init) # W0212(protected-access) # W0221(arguments-differ) @@ -19,7 +18,6 @@ jobs=4 # W0602(global-variable-not-assigned) # W0603(global-statement) # W0611(unused-import) -# W0612(unused-variable) # W0613(unused-argument) # W0621(redefined-outer-name) # W0622(redefined-builtin) @@ -27,7 +25,7 @@ jobs=4 # W0703(broad-except) # W1401(anomalous-backslash-in-string) -disable=C, F, I, R, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401 +disable=C, F, I, R, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401 [REPORTS] diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index 10d6d197..c13020d8 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -55,7 +55,6 @@ class SetHostnameError(Exception): This may happen if we attempt to set the hostname early in cloud-init's init-local timeframe as certain services may not be running yet. """ - pass def handle(name, cfg, cloud, log, _args): diff --git a/cloudinit/distros/networking.py b/cloudinit/distros/networking.py index 75e69df5..10ed249d 100644 --- a/cloudinit/distros/networking.py +++ b/cloudinit/distros/networking.py @@ -92,7 +92,6 @@ class Networking(metaclass=abc.ABCMeta): Examples of non-physical network devices: bonds, bridges, tunnels, loopback devices. """ - pass def is_renamed(self, devname: DeviceName) -> bool: return net.is_renamed(devname) @@ -117,7 +116,6 @@ class Networking(metaclass=abc.ABCMeta): process entirely, if the device already exists.) :type exists: Optional[DeviceName] """ - pass def wait_for_physdevs( self, netcfg: NetworkConfig, *, strict: bool = True @@ -182,7 +180,6 @@ class BSDNetworking(Networking): def settle(self, *, exists=None) -> None: """BSD has no equivalent to `udevadm settle`; noop.""" - pass class LinuxNetworking(Networking): diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index 23be3bdd..b4c4b0c3 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -49,7 +49,5 @@ class Distro(debian.Distro): copy.deepcopy(PREFERRED_NTP_CLIENTS)) return self._preferred_ntp_clients - pass - # vi: ts=4 expandtab diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 0e83685d..7ca7262b 100755 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -29,12 +29,10 @@ class InitramfsNetworkConfigSource(metaclass=abc.ABCMeta): @abc.abstractmethod def is_applicable(self) -> bool: """Is this initramfs config source applicable to the current system?""" - pass @abc.abstractmethod def render_config(self) -> dict: """Render a v1 network config from the initramfs configuration""" - pass class KlibcNetworkConfigSource(InitramfsNetworkConfigSource): diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 9e004688..d1d1255e 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -31,12 +31,10 @@ class InvalidDHCPLeaseFileError(Exception): Current uses are DataSourceAzure and DataSourceEc2 during ephemeral boot to scrape metadata. """ - pass class NoDHCPLeaseError(Exception): """Raised when unable to get a DHCP lease.""" - pass class EphemeralDHCPv4(object): diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 6b9127b6..1986ebdd 100755 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -35,7 +35,6 @@ class ReportingHandler(metaclass=abc.ABCMeta): def flush(self): """Ensure ReportingHandler has published all events""" - pass class LogHandler(ReportingHandler): diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 5e25b956..8b34d047 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -688,7 +688,6 @@ class DataSourceAzure(sources.DataSource): except UrlError: # Teardown our EphemeralDHCPv4 context on failure as we retry self._ephemeral_dhcp_ctx.clean_network() - pass finally: if nl_sock: nl_sock.close() diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 923e3cea..c4d60fff 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -78,7 +78,6 @@ class DataSourceNotFoundException(Exception): class InvalidMetaDataException(Exception): """Raised when metadata is broken, unavailable or disabled.""" - pass def process_instance_metadata(metadata, key_path='', sensitive_keys=()): @@ -511,7 +510,6 @@ class DataSource(metaclass=abc.ABCMeta): (e.g. 'ssh-rsa') and key_value is the key itself (e.g. 'AAAAB3NzaC1y...'). """ - pass def _remap_device(self, short_name): # LP: #611137 diff --git a/cloudinit/sources/helpers/netlink.py b/cloudinit/sources/helpers/netlink.py index d377ae3d..a74a3588 100644 --- a/cloudinit/sources/helpers/netlink.py +++ b/cloudinit/sources/helpers/netlink.py @@ -55,7 +55,6 @@ NetlinkHeader = namedtuple('NetlinkHeader', ['length', 'type', 'flags', 'seq', class NetlinkCreateSocketError(RuntimeError): '''Raised if netlink socket fails during create or bind.''' - pass def create_bound_netlink_socket(): diff --git a/cloudinit/sources/helpers/vmware/imc/config_file.py b/cloudinit/sources/helpers/vmware/imc/config_file.py index 602af078..fc034c95 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_file.py +++ b/cloudinit/sources/helpers/vmware/imc/config_file.py @@ -22,7 +22,6 @@ class ConfigFile(ConfigSource, dict): def __init__(self, filename): self._loadConfigFile(filename) - pass def _insertKey(self, key, val): """ diff --git a/cloudinit/sources/helpers/vmware/imc/config_namespace.py b/cloudinit/sources/helpers/vmware/imc/config_namespace.py index 2f29edd4..5899d8f7 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_namespace.py +++ b/cloudinit/sources/helpers/vmware/imc/config_namespace.py @@ -10,6 +10,5 @@ from .config_source import ConfigSource class ConfigNamespace(ConfigSource): """Specifies the Config Namespace.""" - pass # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py index 2f8ea546..7ec06a9c 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_source.py +++ b/cloudinit/sources/helpers/vmware/imc/config_source.py @@ -8,6 +8,5 @@ class ConfigSource(object): """Specifies a source for the Config Content.""" - pass # vi: ts=4 expandtab diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 69e6b7e1..765f4aab 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -947,7 +947,6 @@ def _pkl_load(fname): except Exception as e: if os.path.isfile(fname): LOG.warning("failed loading pickle in %s: %s", fname, e) - pass # This is allowed so just return nothing successfully loaded... if not pickle_contents: diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py index f1e28a96..a136cf0d 100644 --- a/tests/cloud_tests/platforms/azurecloud/instance.py +++ b/tests/cloud_tests/platforms/azurecloud/instance.py @@ -80,7 +80,6 @@ class AzureCloudInstance(Instance): except CloudError: LOG.debug(('image not found, launching instance with base image, ' 'image_id=%s'), self.image_id) - pass vm_params = { 'name': self.vm_name, @@ -169,7 +168,6 @@ class AzureCloudInstance(Instance): sleep(15) else: LOG.warning('Could not find console log: %s', e) - pass LOG.debug('stopping instance %s', self.image_id) vm_deallocate = \ diff --git a/tests/cloud_tests/platforms/images.py b/tests/cloud_tests/platforms/images.py index 557a5cf6..f047de2e 100644 --- a/tests/cloud_tests/platforms/images.py +++ b/tests/cloud_tests/platforms/images.py @@ -52,6 +52,5 @@ class Image(TargetBase): def destroy(self): """Clean up data associated with image.""" - pass # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/snapshots.py b/tests/cloud_tests/platforms/snapshots.py index 94328982..0f5f8bb6 100644 --- a/tests/cloud_tests/platforms/snapshots.py +++ b/tests/cloud_tests/platforms/snapshots.py @@ -40,6 +40,5 @@ class Snapshot(object): def destroy(self): """Clean up snapshot data.""" - pass # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 2e7c6686..4448e0b5 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -34,7 +34,6 @@ class CloudTestCase(unittest.TestCase): @classmethod def maybeSkipTest(cls): """Present to allow subclasses to override and raise a skipTest.""" - pass def assertPackageInstalled(self, name, version=None): """Check dpkg-query --show output for matching package name. diff --git a/tests/unittests/test_datasource/test_ibmcloud.py b/tests/unittests/test_datasource/test_ibmcloud.py index 0b54f585..9013ae9f 100644 --- a/tests/unittests/test_datasource/test_ibmcloud.py +++ b/tests/unittests/test_datasource/test_ibmcloud.py @@ -15,13 +15,6 @@ mock = test_helpers.mock D_PATH = "cloudinit.sources.DataSourceIBMCloud." -class TestIBMCloud(test_helpers.CiTestCase): - """Test the datasource.""" - def setUp(self): - super(TestIBMCloud, self).setUp() - pass - - @mock.patch(D_PATH + "_is_xen", return_value=True) @mock.patch(D_PATH + "_is_ibm_provisioning") @mock.patch(D_PATH + "util.blkid") diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 6814030e..9f11fd5c 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -349,7 +349,6 @@ class TestReportingEventStack(TestCase): with parent: with child: pass - pass self.assertEqual(report_start.call_count, 0) self.assertEqual(report_finish.call_count, 0) -- cgit v1.2.3 From db86753f81af73826158c9522f2521f210300e2b Mon Sep 17 00:00:00 2001 From: Eduardo Otubo Date: Mon, 24 Aug 2020 15:34:24 +0200 Subject: DHCP sandboxing failing on noexec mounted /var/tmp (#521) * DHCP sandboxing failing on noexec mounted /var/tmp If /var/tmp is mounted with noexec option the DHCP sandboxing will fail with Permission Denied. This patch simply avoids this error by checking the exec permission updating the dhcp path in negative case. rhbz: https://bugzilla.redhat.com/show_bug.cgi?id=1857309 Signed-off-by: Eduardo Otubo * Replacing with os.* calls * Adding test and removing isfile() useless call. Co-authored-by: Rick Harding --- cloudinit/net/dhcp.py | 6 ++++++ cloudinit/net/tests/test_dhcp.py | 46 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) (limited to 'cloudinit/net/dhcp.py') diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index d1d1255e..9230cf7a 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -220,6 +220,12 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir, dhcp_log_func=None): pid_file = os.path.join(cleandir, 'dhclient.pid') lease_file = os.path.join(cleandir, 'dhcp.leases') + # In some cases files in /var/tmp may not be executable, launching dhclient + # from there will certainly raise 'Permission denied' error. Try launching + # the original dhclient instead. + if not os.access(sandbox_dhclient_cmd, os.X_OK): + sandbox_dhclient_cmd = dhclient_cmd_path + # ISC dhclient needs the interface up to send initial discovery packets. # Generally dhclient relies on dhclient-script PREINIT action to bring the # link up before attempting discovery. Since we are using -sf /bin/true, diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index 7987f9d1..74cf4b94 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -409,6 +409,52 @@ class TestDHCPDiscoveryClean(CiTestCase): 'eth9', '-sf', '/bin/true'], capture=True)]) m_kill.assert_has_calls([mock.call(my_pid, signal.SIGKILL)]) + @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid') + @mock.patch('cloudinit.net.dhcp.os.kill') + @mock.patch('cloudinit.net.dhcp.subp.subp') + def test_dhcp_discovery_outside_sandbox(self, m_subp, m_kill, m_getppid): + """dhcp_discovery brings up the interface and runs dhclient. + + It also returns the parsed dhcp.leases file generated in the sandbox. + """ + m_subp.return_value = ('', '') + tmpdir = self.tmp_dir() + dhclient_script = os.path.join(tmpdir, 'dhclient.orig') + script_content = '#!/bin/bash\necho fake-dhclient' + write_file(dhclient_script, script_content, mode=0o755) + lease_content = dedent(""" + lease { + interface "eth9"; + fixed-address 192.168.2.74; + option subnet-mask 255.255.255.0; + option routers 192.168.2.1; + } + """) + lease_file = os.path.join(tmpdir, 'dhcp.leases') + write_file(lease_file, lease_content) + pid_file = os.path.join(tmpdir, 'dhclient.pid') + my_pid = 1 + write_file(pid_file, "%d\n" % my_pid) + m_getppid.return_value = 1 # Indicate that dhclient has daemonized + + with mock.patch('os.access', return_value=False): + self.assertCountEqual( + [{'interface': 'eth9', 'fixed-address': '192.168.2.74', + 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], + dhcp_discovery(dhclient_script, 'eth9', tmpdir)) + # dhclient script got copied + with open(os.path.join(tmpdir, 'dhclient.orig')) as stream: + self.assertEqual(script_content, stream.read()) + # Interface was brought up before dhclient called from sandbox + m_subp.assert_has_calls([ + mock.call( + ['ip', 'link', 'set', 'dev', 'eth9', 'up'], capture=True), + mock.call( + [os.path.join(tmpdir, 'dhclient.orig'), '-1', '-v', '-lf', + lease_file, '-pf', os.path.join(tmpdir, 'dhclient.pid'), + 'eth9', '-sf', '/bin/true'], capture=True)]) + m_kill.assert_has_calls([mock.call(my_pid, signal.SIGKILL)]) + @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid') @mock.patch('cloudinit.net.dhcp.os.kill') @mock.patch('cloudinit.net.dhcp.subp.subp') -- cgit v1.2.3 From 07104504ab5b30efd2d1f7a8c36effe18b8e5fe0 Mon Sep 17 00:00:00 2001 From: Paride Legovini Date: Tue, 25 Aug 2020 17:21:18 +0200 Subject: tox: bump the pylint version to 2.6.0 in the default run (#544) Changes: tox: bump the pylint version to 2.6.0 in the default run Fix pylint 2.6.0 W0707 warnings (raise-missing-from) --- cloudinit/analyze/show.py | 2 +- cloudinit/cmd/devel/make_mime.py | 6 ++-- cloudinit/config/cc_disk_setup.py | 36 ++++++++++++++-------- cloudinit/config/cc_growpart.py | 8 ++--- cloudinit/config/cc_power_state_change.py | 12 +++++--- cloudinit/config/cc_rsyslog.py | 6 ++-- cloudinit/config/cc_set_hostname.py | 2 +- cloudinit/config/cc_ubuntu_advantage.py | 2 +- cloudinit/config/schema.py | 2 +- cloudinit/distros/__init__.py | 5 +-- cloudinit/distros/arch.py | 4 +-- cloudinit/distros/parsers/resolv_conf.py | 7 +++-- cloudinit/gpg.py | 5 +-- cloudinit/handlers/jinja_template.py | 3 +- cloudinit/net/__init__.py | 7 +++-- cloudinit/net/cmdline.py | 4 +-- cloudinit/net/dhcp.py | 4 +-- cloudinit/net/network_state.py | 19 +++++++----- cloudinit/sources/DataSourceAzure.py | 2 +- cloudinit/sources/DataSourceEc2.py | 8 +++-- cloudinit/sources/DataSourceIBMCloud.py | 3 +- cloudinit/sources/DataSourceMAAS.py | 3 +- cloudinit/sources/DataSourceOpenNebula.py | 13 +++++--- cloudinit/sources/DataSourceOpenStack.py | 4 +-- cloudinit/sources/DataSourceSmartOS.py | 4 ++- cloudinit/sources/helpers/azure.py | 34 +++++++++++--------- cloudinit/sources/helpers/netlink.py | 2 +- cloudinit/sources/helpers/openstack.py | 34 +++++++++++--------- cloudinit/subp.py | 3 +- cloudinit/url_helper.py | 6 ++-- cloudinit/util.py | 8 ++--- tests/cloud_tests/platforms/azurecloud/instance.py | 7 +++-- tests/cloud_tests/platforms/azurecloud/platform.py | 23 +++++++++----- tests/cloud_tests/platforms/ec2/instance.py | 4 +-- tests/cloud_tests/platforms/ec2/platform.py | 18 ++++++----- tests/cloud_tests/platforms/lxd/instance.py | 3 +- tests/cloud_tests/platforms/platforms.py | 6 ++-- tests/cloud_tests/testcases/__init__.py | 6 ++-- tools/mock-meta.py | 14 +++++---- tox.ini | 2 +- 40 files changed, 205 insertions(+), 136 deletions(-) (limited to 'cloudinit/net/dhcp.py') diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py index 0c825b23..01a4d3e5 100644 --- a/cloudinit/analyze/show.py +++ b/cloudinit/analyze/show.py @@ -267,7 +267,7 @@ def gather_timestamps_using_systemd(): except OSError as err: raise RuntimeError('Could not determine container boot ' 'time from /proc/1/cmdline. ({})' - .format(err)) + .format(err)) from err status = CONTAINER_CODE else: status = FAIL_CODE diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index 77e10540..4e6a5778 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -22,8 +22,10 @@ def file_content_type(text): try: filename, content_type = text.split(":", 1) return (open(filename, 'r'), filename, content_type.strip()) - except ValueError: - raise argparse.ArgumentError(text, "Invalid value for %r" % (text)) + except ValueError as e: + raise argparse.ArgumentError( + text, "Invalid value for %r" % (text) + ) from e def get_parser(parser=None): diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index d957cfe3..a7bdc703 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -251,7 +251,9 @@ def enumerate_disk(device, nodeps=False): try: info, _err = subp.subp(lsblk_cmd) except Exception as e: - raise Exception("Failed during disk check for %s\n%s" % (device, e)) + raise Exception( + "Failed during disk check for %s\n%s" % (device, e) + ) from e parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0] @@ -313,7 +315,9 @@ def check_fs(device): try: out, _err = subp.subp(blkid_cmd, rcs=[0, 2]) except Exception as e: - raise Exception("Failed during disk check for %s\n%s" % (device, e)) + raise Exception( + "Failed during disk check for %s\n%s" % (device, e) + ) from e if out: if len(out.splitlines()) == 1: @@ -428,8 +432,8 @@ def get_dyn_func(*args): else: return globals()[func_name] - except KeyError: - raise Exception("No such function %s to call!" % func_name) + except KeyError as e: + raise Exception("No such function %s to call!" % func_name) from e def get_hdd_size(device): @@ -437,7 +441,7 @@ def get_hdd_size(device): size_in_bytes, _ = subp.subp([BLKDEV_CMD, '--getsize64', device]) sector_size, _ = subp.subp([BLKDEV_CMD, '--getss', device]) except Exception as e: - raise Exception("Failed to get %s size\n%s" % (device, e)) + raise Exception("Failed to get %s size\n%s" % (device, e)) from e return int(size_in_bytes) / int(sector_size) @@ -455,8 +459,9 @@ def check_partition_mbr_layout(device, layout): try: out, _err = subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: - raise Exception("Error running partition command on %s\n%s" % ( - device, e)) + raise Exception( + "Error running partition command on %s\n%s" % (device, e) + ) from e found_layout = [] for line in out.splitlines(): @@ -485,8 +490,9 @@ def check_partition_gpt_layout(device, layout): try: out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV) except Exception as e: - raise Exception("Error running partition command on %s\n%s" % ( - device, e)) + raise Exception( + "Error running partition command on %s\n%s" % (device, e) + ) from e out_lines = iter(out.splitlines()) # Skip header. Output looks like: @@ -657,8 +663,10 @@ def purge_disk(device): try: LOG.info("Purging filesystem on /dev/%s", d['name']) subp.subp(wipefs_cmd) - except Exception: - raise Exception("Failed FS purge of /dev/%s" % d['name']) + except Exception as e: + raise Exception( + "Failed FS purge of /dev/%s" % d['name'] + ) from e purge_disk_ptable(device) @@ -700,7 +708,9 @@ def exec_mkpart_mbr(device, layout): try: subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: - raise Exception("Failed to partition device %s\n%s" % (device, e)) + raise Exception( + "Failed to partition device %s\n%s" % (device, e) + ) from e read_parttbl(device) @@ -997,6 +1007,6 @@ def mkfs(fs_cfg): try: subp.subp(fs_cmd, shell=shell) except Exception as e: - raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) + raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) from e # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index c5d93f81..237c3d02 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -148,14 +148,14 @@ class ResizeGrowPart(object): if e.exit_code != 1: util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)", diskdev, partnum) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e return (before, before) try: subp.subp(["growpart", diskdev, partnum]) except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: growpart %s %s", diskdev, partnum) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e return (before, get_size(partdev)) @@ -187,14 +187,14 @@ class ResizeGpart(object): except subp.ProcessExecutionError as e: if e.exit_code != 0: util.logexc(LOG, "Failed: gpart recover %s", diskdev) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e before = get_size(partdev) try: subp.subp(["gpart", "resize", "-i", partnum, diskdev]) except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e # Since growing the FS requires a reboot, make sure we reboot # first when this module has finished. diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index ab953a0d..6fcb8a7d 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -196,10 +196,11 @@ def load_power_state(cfg, distro_name): try: delay = convert_delay(delay, fmt=fmt, scale=scale) - except ValueError: + except ValueError as e: raise TypeError( "power_state[delay] must be 'now' or '+m' (minutes)." - " found '%s'." % delay) + " found '%s'." % delay + ) from e args = command + [delay] if message: @@ -207,9 +208,10 @@ def load_power_state(cfg, distro_name): try: timeout = float(pstate.get('timeout', 30.0)) - except ValueError: - raise ValueError("failed to convert timeout '%s' to float." % - pstate['timeout']) + except ValueError as e: + raise ValueError( + "failed to convert timeout '%s' to float." % pstate['timeout'] + ) from e condition = pstate.get("condition", True) if not isinstance(condition, (str, list, bool)): diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 1354885a..2a2bc931 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -347,8 +347,10 @@ class SyslogRemotesLine(object): if self.port: try: int(self.port) - except ValueError: - raise ValueError("port '%s' is not an integer" % self.port) + except ValueError as e: + raise ValueError( + "port '%s' is not an integer" % self.port + ) from e if not self.addr: raise ValueError("address is required") diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index c13020d8..1d23d80d 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -85,7 +85,7 @@ def handle(name, cfg, cloud, log, _args): except Exception as e: msg = "Failed to set the hostname to %s (%s)" % (fqdn, hostname) util.logexc(log, msg) - raise SetHostnameError("%s: %s" % (msg, e)) + raise SetHostnameError("%s: %s" % (msg, e)) from e write_json(prev_fn, {'hostname': hostname, 'fqdn': fqdn}) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py index 35ded5db..d61dc655 100644 --- a/cloudinit/config/cc_ubuntu_advantage.py +++ b/cloudinit/config/cc_ubuntu_advantage.py @@ -115,7 +115,7 @@ def configure_ua(token=None, enable=None): msg = 'Failure attaching Ubuntu Advantage:\n{error}'.format( error=str(e)) util.logexc(LOG, msg) - raise RuntimeError(msg) + raise RuntimeError(msg) from e enable_errors = [] for service in enable: try: diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 2d8c7577..8a966aee 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -210,7 +210,7 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): error = SchemaValidationError(errors) if annotate: print(annotated_cloudconfig_file({}, content, error.schema_errors)) - raise error + raise error from e try: validate_cloudconfig_schema( cloudconfig, schema, strict=True) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index effb4276..2537608f 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -607,10 +607,11 @@ class Distro(metaclass=abc.ABCMeta): lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name]) try: cmd = next(tool for tool in lock_tools if subp.which(tool[0])) - except StopIteration: + except StopIteration as e: raise RuntimeError(( "Unable to lock user account '%s'. No tools available. " - " Tried: %s.") % (name, [c[0] for c in lock_tools])) + " Tried: %s.") % (name, [c[0] for c in lock_tools]) + ) from e try: subp.subp(cmd) except Exception as e: diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 038aa9ac..967be168 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -61,9 +61,9 @@ class Distro(distros.Distro): def _write_network_config(self, netconfig): try: return self._supported_write_network_config(netconfig) - except RendererNotFoundError: + except RendererNotFoundError as e: # Fall back to old _write_network - raise NotImplementedError + raise NotImplementedError from e def _write_network(self, settings): entries = net_util.translate_network(settings) diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py index 299d54b5..62929d03 100644 --- a/cloudinit/distros/parsers/resolv_conf.py +++ b/cloudinit/distros/parsers/resolv_conf.py @@ -150,9 +150,10 @@ class ResolvConf(object): tail = '' try: (cfg_opt, cfg_values) = head.split(None, 1) - except (IndexError, ValueError): - raise IOError("Incorrectly formatted resolv.conf line %s" - % (i + 1)) + except (IndexError, ValueError) as e: + raise IOError( + "Incorrectly formatted resolv.conf line %s" % (i + 1) + ) from e if cfg_opt not in ['nameserver', 'domain', 'search', 'sortlist', 'options']: raise IOError("Unexpected resolv.conf option %s" % (cfg_opt)) diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py index 72b5ac59..be0ca0ea 100644 --- a/cloudinit/gpg.py +++ b/cloudinit/gpg.py @@ -63,10 +63,11 @@ def recv_key(key, keyserver, retries=(1, 1)): "Import failed with exit code %d, will try again in %ss", error.exit_code, naplen) time.sleep(naplen) - except StopIteration: + except StopIteration as e: raise ValueError( ("Failed to import key '%s' from keyserver '%s' " - "after %d tries: %s") % (key, keyserver, trynum, error)) + "after %d tries: %s") % (key, keyserver, trynum, error) + ) from e def delete_key(key): diff --git a/cloudinit/handlers/jinja_template.py b/cloudinit/handlers/jinja_template.py index ce3accf6..aadfbf86 100644 --- a/cloudinit/handlers/jinja_template.py +++ b/cloudinit/handlers/jinja_template.py @@ -83,7 +83,8 @@ def render_jinja_payload_from_file( if e.errno == EACCES: raise RuntimeError( 'Cannot render jinja template vars. No read permission on' - " '%s'. Try sudo" % instance_data_file) + " '%s'. Try sudo" % instance_data_file + ) from e rendered_payload = render_jinja_payload( payload, payload_fn, instance_data, debug) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 322af77b..e233149a 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -506,7 +506,9 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): try: _rename_interfaces(extract_physdevs(netcfg)) except RuntimeError as e: - raise RuntimeError('Failed to apply network config names: %s' % e) + raise RuntimeError( + 'Failed to apply network config names: %s' % e + ) from e def interface_has_own_mac(ifname, strict=False): @@ -965,7 +967,8 @@ class EphemeralIPv4Network(object): self.prefix = mask_to_net_prefix(prefix_or_mask) except ValueError as e: raise ValueError( - 'Cannot setup network: {0}'.format(e)) + 'Cannot setup network: {0}'.format(e) + ) from e self.connectivity_url = connectivity_url self.interface = interface diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 7ca7262b..cc8dc17b 100755 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -112,8 +112,8 @@ def _klibc_to_config_entry(content, mac_addrs=None): data = util.load_shell_content(content) try: name = data['DEVICE'] if 'DEVICE' in data else data['DEVICE6'] - except KeyError: - raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data") + except KeyError as e: + raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data") from e # ipconfig on precise does not write PROTO # IPv6 config gives us IPV6PROTO, not PROTO. diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 9230cf7a..4394c68b 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -82,8 +82,8 @@ class EphemeralDHCPv4(object): try: leases = maybe_perform_dhcp_discovery( self.iface, self.dhcp_log_func) - except InvalidDHCPLeaseFileError: - raise NoDHCPLeaseError() + except InvalidDHCPLeaseFileError as e: + raise NoDHCPLeaseError() from e if not leases: raise NoDHCPLeaseError() self.lease = leases[-1] diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 7bfe8be0..b2f7d31e 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -297,9 +297,10 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): command_type = command['type'] try: handler = self.command_handlers[command_type] - except KeyError: - raise RuntimeError("No handler found for" - " command '%s'" % command_type) + except KeyError as e: + raise RuntimeError( + "No handler found for command '%s'" % command_type + ) from e try: handler(self, command) except InvalidCommand: @@ -316,9 +317,10 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): continue try: handler = self.command_handlers[command_type] - except KeyError: - raise RuntimeError("No handler found for" - " command '%s'" % command_type) + except KeyError as e: + raise RuntimeError( + "No handler found for command '%s'" % command_type + ) from e try: handler(self, command) self._v2_common(command) @@ -914,9 +916,10 @@ def _normalize_route(route): if metric: try: normal_route['metric'] = int(metric) - except ValueError: + except ValueError as e: raise TypeError( - 'Route config metric {} is not an integer'.format(metric)) + 'Route config metric {} is not an integer'.format(metric) + ) from e return normal_route diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 86cc7c28..f3c6452b 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -1147,7 +1147,7 @@ def read_azure_ovf(contents): except Exception as e: error_str = "Invalid ovf-env.xml: %s" % e report_diagnostic_event(error_str) - raise BrokenAzureDataSource(error_str) + raise BrokenAzureDataSource(error_str) from e results = find_child(dom.documentElement, lambda n: n.localName == "ProvisioningSection") diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 355b4e2f..1d09c12a 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -617,9 +617,11 @@ def parse_strict_mode(cfgval): if sleep: try: sleep = int(sleep) - except ValueError: - raise ValueError("Invalid sleep '%s' in strict_id setting '%s': " - "not an integer" % (sleep, cfgval)) + except ValueError as e: + raise ValueError( + "Invalid sleep '%s' in strict_id setting '%s': not an integer" + % (sleep, cfgval) + ) from e else: sleep = None diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py index d2aa9a58..8d196185 100644 --- a/cloudinit/sources/DataSourceIBMCloud.py +++ b/cloudinit/sources/DataSourceIBMCloud.py @@ -303,7 +303,8 @@ def read_md(): except sources.BrokenMetadata as e: raise RuntimeError( "Failed reading IBM config disk (platform=%s path=%s): %s" % - (platform, path, e)) + (platform, path, e) + ) from e ret.update(results) return ret diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index c80f70c2..9156925f 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -226,7 +226,8 @@ def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None, except url_helper.UrlError as e: if e.code == 404 and not optional: raise MAASSeedDirMalformed( - "Missing required %s: %s" % (path, e)) + "Missing required %s: %s" % (path, e) + ) from e elif e.code != 404: raise e diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 12b1f94f..45481938 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -399,18 +399,23 @@ def read_context_disk_dir(source_dir, distro, asuser=None): if asuser is not None: try: pwd.getpwnam(asuser) - except KeyError: + except KeyError as e: raise BrokenContextDiskDir( "configured user '{user}' does not exist".format( - user=asuser)) + user=asuser) + ) from e try: path = os.path.join(source_dir, 'context.sh') content = util.load_file(path) context = parse_shell_config(content, asuser=asuser) except subp.ProcessExecutionError as e: - raise BrokenContextDiskDir("Error processing context.sh: %s" % (e)) + raise BrokenContextDiskDir( + "Error processing context.sh: %s" % (e) + ) from e except IOError as e: - raise NonContextDiskDir("Error reading context.sh: %s" % (e)) + raise NonContextDiskDir( + "Error reading context.sh: %s" % (e) + ) from e else: raise NonContextDiskDir("Missing context.sh") diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index bf539091..d4b43f44 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -194,10 +194,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'timeout': url_params.timeout_seconds}) except openstack.NonReadable as e: raise sources.InvalidMetaDataException(str(e)) - except (openstack.BrokenMetadata, IOError): + except (openstack.BrokenMetadata, IOError) as e: msg = 'Broken metadata address {addr}'.format( addr=self.metadata_address) - raise sources.InvalidMetaDataException(msg) + raise sources.InvalidMetaDataException(msg) from e return result diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 843b3a2a..f1f903bc 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -413,7 +413,9 @@ class JoyentMetadataClient(object): response.append(byte) except OSError as exc: if exc.errno == errno.EAGAIN: - raise JoyentMetadataTimeoutException(msg % as_ascii()) + raise JoyentMetadataTimeoutException( + msg % as_ascii() + ) from exc raise def _write(self, msg): diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 6156c75b..b968a96f 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -98,8 +98,10 @@ def get_boot_telemetry(): LOG.debug("Collecting boot telemetry") try: kernel_start = float(time.time()) - float(util.uptime()) - except ValueError: - raise RuntimeError("Failed to determine kernel start timestamp") + except ValueError as e: + raise RuntimeError( + "Failed to determine kernel start timestamp" + ) from e try: out, _ = subp.subp(['/bin/systemctl', @@ -116,12 +118,13 @@ def get_boot_telemetry(): user_start = kernel_start + (float(tsm) / 1000000) except subp.ProcessExecutionError as e: - raise RuntimeError("Failed to get UserspaceTimestampMonotonic: %s" - % e) + raise RuntimeError( + "Failed to get UserspaceTimestampMonotonic: %s" % e + ) from e except ValueError as e: - raise RuntimeError("Failed to parse " - "UserspaceTimestampMonotonic from systemd: %s" - % e) + raise RuntimeError( + "Failed to parse UserspaceTimestampMonotonic from systemd: %s" % e + ) from e try: out, _ = subp.subp(['/bin/systemctl', 'show', @@ -137,12 +140,14 @@ def get_boot_telemetry(): cloudinit_activation = kernel_start + (float(tsm) / 1000000) except subp.ProcessExecutionError as e: - raise RuntimeError("Failed to get InactiveExitTimestampMonotonic: %s" - % e) + raise RuntimeError( + "Failed to get InactiveExitTimestampMonotonic: %s" % e + ) from e except ValueError as e: - raise RuntimeError("Failed to parse " - "InactiveExitTimestampMonotonic from systemd: %s" - % e) + raise RuntimeError( + "Failed to parse InactiveExitTimestampMonotonic from systemd: %s" + % e + ) from e evt = events.ReportingEvent( BOOT_EVENT_TYPE, 'boot-telemetry', @@ -642,9 +647,10 @@ class WALinuxAgentShim: try: name = os.path.basename(hook_file).replace('.json', '') dhcp_options[name] = json.loads(util.load_file((hook_file))) - except ValueError: + except ValueError as e: raise ValueError( - '{_file} is not valid JSON data'.format(_file=hook_file)) + '{_file} is not valid JSON data'.format(_file=hook_file) + ) from e return dhcp_options @staticmethod diff --git a/cloudinit/sources/helpers/netlink.py b/cloudinit/sources/helpers/netlink.py index a74a3588..c2ad587b 100644 --- a/cloudinit/sources/helpers/netlink.py +++ b/cloudinit/sources/helpers/netlink.py @@ -74,7 +74,7 @@ def create_bound_netlink_socket(): netlink_socket.setblocking(0) except socket.error as e: msg = "Exception during netlink socket create: %s" % e - raise NetlinkCreateSocketError(msg) + raise NetlinkCreateSocketError(msg) from e LOG.debug("Created netlink socket") return netlink_socket diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 1050efb0..65e020c5 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -280,8 +280,9 @@ class BaseReader(metaclass=abc.ABCMeta): try: data = translator(data) except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to process path %s: %s" % (path, e) + ) from e if found: results[name] = data @@ -291,8 +292,9 @@ class BaseReader(metaclass=abc.ABCMeta): try: metadata['random_seed'] = base64.b64decode(random_seed) except (ValueError, TypeError) as e: - raise BrokenMetadata("Badly formatted metadata" - " random_seed entry: %s" % e) + raise BrokenMetadata( + "Badly formatted metadata random_seed entry: %s" % e + ) from e # load any files that were provided files = {} @@ -304,8 +306,9 @@ class BaseReader(metaclass=abc.ABCMeta): try: files[path] = self._read_content_path(item) except Exception as e: - raise BrokenMetadata("Failed to read provided " - "file %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to read provided file %s: %s" % (path, e) + ) from e results['files'] = files # The 'network_config' item in metadata is a content pointer @@ -317,8 +320,9 @@ class BaseReader(metaclass=abc.ABCMeta): content = self._read_content_path(net_item, decode=True) results['network_config'] = content except IOError as e: - raise BrokenMetadata("Failed to read network" - " configuration: %s" % (e)) + raise BrokenMetadata( + "Failed to read network configuration: %s" % (e) + ) from e # To openstack, user can specify meta ('nova boot --meta=key=value') # and those will appear under metadata['meta']. @@ -370,8 +374,9 @@ class ConfigDriveReader(BaseReader): try: return util.load_json(self._path_read(path)) except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to process path %s: %s" % (path, e) + ) from e def read_v1(self): """Reads a version 1 formatted location. @@ -395,16 +400,17 @@ class ConfigDriveReader(BaseReader): path = found[name] try: contents = self._path_read(path) - except IOError: - raise BrokenMetadata("Failed to read: %s" % path) + except IOError as e: + raise BrokenMetadata("Failed to read: %s" % path) from e try: # Disable not-callable pylint check; pylint isn't able to # determine that every member of FILES_V1 has a callable in # the appropriate position md[key] = translator(contents) # pylint: disable=E1102 except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to process path %s: %s" % (path, e) + ) from e else: md[key] = copy.deepcopy(default) diff --git a/cloudinit/subp.py b/cloudinit/subp.py index 804ef3ca..3e4efa42 100644 --- a/cloudinit/subp.py +++ b/cloudinit/subp.py @@ -262,7 +262,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True, raise ProcessExecutionError( cmd=args, reason=e, errno=e.errno, stdout="-" if decode else b"-", - stderr="-" if decode else b"-") + stderr="-" if decode else b"-" + ) from e finally: if devnull_fp: devnull_fp.close() diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index f3c0cf9c..caa88435 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -95,7 +95,7 @@ def read_file_or_url(url, **kwargs): code = e.errno if e.errno == ENOENT: code = NOT_FOUND - raise UrlError(cause=e, code=code, headers=None, url=url) + raise UrlError(cause=e, code=code, headers=None, url=url) from e return FileResponse(file_path, contents=contents) else: return readurl(url, **kwargs) @@ -575,8 +575,8 @@ def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret, timestamp=None): try: import oauthlib.oauth1 as oauth1 - except ImportError: - raise NotImplementedError('oauth support is not available') + except ImportError as e: + raise NotImplementedError('oauth support is not available') from e if timestamp: timestamp = str(timestamp) diff --git a/cloudinit/util.py b/cloudinit/util.py index dd263803..cf9e349f 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -359,7 +359,7 @@ def decomp_gzip(data, quiet=True, decode=True): if quiet: return data else: - raise DecompressionError(str(e)) + raise DecompressionError(str(e)) from e def extract_usergroup(ug_pair): @@ -1363,7 +1363,7 @@ def chownbyname(fname, user=None, group=None): if group: gid = grp.getgrnam(group).gr_gid except KeyError as e: - raise OSError("Unknown user or group: %s" % (e)) + raise OSError("Unknown user or group: %s" % (e)) from e chownbyid(fname, uid, gid) @@ -2387,8 +2387,8 @@ def human2bytes(size): try: num = float(num) - except ValueError: - raise ValueError("'%s' is not valid input." % size_in) + except ValueError as e: + raise ValueError("'%s' is not valid input." % size_in) from e if num < 0: raise ValueError("'%s': cannot be negative" % size_in) diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py index a136cf0d..eedbaae8 100644 --- a/tests/cloud_tests/platforms/azurecloud/instance.py +++ b/tests/cloud_tests/platforms/azurecloud/instance.py @@ -134,9 +134,10 @@ class AzureCloudInstance(Instance): self.vm_name, vm_params) LOG.debug('creating instance %s from image_id=%s', self.vm_name, self.image_id) - except CloudError: - raise RuntimeError('failed creating instance:\n{}'.format( - traceback.format_exc())) + except CloudError as e: + raise RuntimeError( + 'failed creating instance:\n{}'.format(traceback.format_exc()) + ) from e if wait: self.instance.wait() diff --git a/tests/cloud_tests/platforms/azurecloud/platform.py b/tests/cloud_tests/platforms/azurecloud/platform.py index cb62a74b..a664f612 100644 --- a/tests/cloud_tests/platforms/azurecloud/platform.py +++ b/tests/cloud_tests/platforms/azurecloud/platform.py @@ -59,9 +59,12 @@ class AzureCloudPlatform(Platform): self.vnet = self._create_vnet() self.subnet = self._create_subnet() self.nic = self._create_nic() - except CloudError: - raise RuntimeError('failed creating a resource:\n{}'.format( - traceback.format_exc())) + except CloudError as e: + raise RuntimeError( + 'failed creating a resource:\n{}'.format( + traceback.format_exc() + ) + ) from e def create_instance(self, properties, config, features, image_id, user_data=None): @@ -105,8 +108,10 @@ class AzureCloudPlatform(Platform): if image_id.find('__') > 0: image_id = image_id.split('__')[1] LOG.debug('image_id shortened to %s', image_id) - except KeyError: - raise RuntimeError('no images found for %s' % img_conf['release']) + except KeyError as e: + raise RuntimeError( + 'no images found for %s' % img_conf['release'] + ) from e return AzureCloudImage(self, img_conf, image_id) @@ -140,9 +145,11 @@ class AzureCloudPlatform(Platform): secret=azure_creds['clientSecret'], tenant=azure_creds['tenantId']) return credentials, subscription_id - except KeyError: - raise RuntimeError('Please configure Azure service principal' - ' credentials in %s' % cred_file) + except KeyError as e: + raise RuntimeError( + 'Please configure Azure service principal' + ' credentials in %s' % cred_file + ) from e def _create_resource_group(self): """Create resource group""" diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py index ab6037b1..d2e84047 100644 --- a/tests/cloud_tests/platforms/ec2/instance.py +++ b/tests/cloud_tests/platforms/ec2/instance.py @@ -49,11 +49,11 @@ class EC2Instance(Instance): # OutputBytes comes from platform._decode_console_output_as_bytes response = self.instance.console_output() return response['OutputBytes'] - except KeyError: + except KeyError as e: if 'Output' in response: msg = ("'OutputBytes' did not exist in console_output() but " "'Output' did: %s..." % response['Output'][0:128]) - raise util.PlatformError('console_log', msg) + raise util.PlatformError('console_log', msg) from e return ('No Console Output [%s]' % self.instance).encode() def destroy(self): diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py index 7a3d0fe0..b61a2ffb 100644 --- a/tests/cloud_tests/platforms/ec2/platform.py +++ b/tests/cloud_tests/platforms/ec2/platform.py @@ -35,12 +35,14 @@ class EC2Platform(Platform): self.ec2_resource = b3session.resource('ec2') self.ec2_region = b3session.region_name self.key_name = self._upload_public_key(config) - except botocore.exceptions.NoRegionError: + except botocore.exceptions.NoRegionError as e: raise RuntimeError( - 'Please configure default region in $HOME/.aws/config') - except botocore.exceptions.NoCredentialsError: + 'Please configure default region in $HOME/.aws/config' + ) from e + except botocore.exceptions.NoCredentialsError as e: raise RuntimeError( - 'Please configure ec2 credentials in $HOME/.aws/credentials') + 'Please configure ec2 credentials in $HOME/.aws/credentials' + ) from e self.vpc = self._create_vpc() self.internet_gateway = self._create_internet_gateway() @@ -125,8 +127,10 @@ class EC2Platform(Platform): try: image_ami = image['id'] - except KeyError: - raise RuntimeError('No images found for %s!' % img_conf['release']) + except KeyError as e: + raise RuntimeError( + 'No images found for %s!' % img_conf['release'] + ) from e LOG.debug('found image: %s', image_ami) image = EC2Image(self, img_conf, image_ami) @@ -195,7 +199,7 @@ class EC2Platform(Platform): CidrBlock=self.ipv4_cidr, AmazonProvidedIpv6CidrBlock=True) except botocore.exceptions.ClientError as e: - raise RuntimeError(e) + raise RuntimeError(e) from e vpc.wait_until_available() self._tag_resource(vpc) diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py index b27b9848..2b973a08 100644 --- a/tests/cloud_tests/platforms/lxd/instance.py +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -175,7 +175,8 @@ class LXDInstance(Instance): raise PlatformError( "console log", "Console log failed [%d]: stdout=%s stderr=%s" % ( - e.exit_code, e.stdout, e.stderr)) + e.exit_code, e.stdout, e.stderr) + ) from e def reboot(self, wait=True): """Reboot instance.""" diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py index 58f65e52..ac3b6563 100644 --- a/tests/cloud_tests/platforms/platforms.py +++ b/tests/cloud_tests/platforms/platforms.py @@ -74,8 +74,10 @@ class Platform(object): try: return tmirror.json_entries[0] - except IndexError: - raise RuntimeError('no images found with filter: %s' % img_filter) + except IndexError as e: + raise RuntimeError( + 'no images found with filter: %s' % img_filter + ) from e class FilterMirror(mirrors.BasicMirrorWriter): diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py index e8c371ca..bb9785d3 100644 --- a/tests/cloud_tests/testcases/__init__.py +++ b/tests/cloud_tests/testcases/__init__.py @@ -21,8 +21,10 @@ def discover_test(test_name): config.name_sanitize(test_name)) try: testmod = importlib.import_module(testmod_name) - except NameError: - raise ValueError('no test verifier found at: {}'.format(testmod_name)) + except NameError as e: + raise ValueError( + 'no test verifier found at: {}'.format(testmod_name) + ) from e found = [mod for name, mod in inspect.getmembers(testmod) if (inspect.isclass(mod) diff --git a/tools/mock-meta.py b/tools/mock-meta.py index a58e0260..9dd067b9 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -258,12 +258,14 @@ class MetaDataHandler(object): try: key_id = int(mybe_key) key_name = key_ids[key_id] - except ValueError: - raise WebException(hclient.BAD_REQUEST, - "%s: not an integer" % mybe_key) - except IndexError: - raise WebException(hclient.NOT_FOUND, - "Unknown key id %r" % mybe_key) + except ValueError as e: + raise WebException( + hclient.BAD_REQUEST, "%s: not an integer" % mybe_key + ) from e + except IndexError as e: + raise WebException( + hclient.NOT_FOUND, "Unknown key id %r" % mybe_key + ) from e # Extract the possible sub-params result = traverse(nparams[1:], { "openssh-key": "\n".join(avail_keys[key_name]), diff --git a/tox.ini b/tox.ini index f619dbf5..a92c63e0 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ setenv = basepython = python3 deps = # requirements - pylint==2.5.3 + pylint==2.6.0 # test-requirements because unit tests are now present in cloudinit tree -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt -- cgit v1.2.3