From 3c551f6ebc12f7729a2755c89b19b9000e27cc88 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 8 Jun 2020 12:49:12 -0400 Subject: Move subp into its own module. (#416) This was painful, but it finishes a TODO from cloudinit/subp.py. It moves the following from util to subp: ProcessExecutionError subp which target_path I moved subp_blob_in_tempfile into cc_chef, which is its only caller. That saved us from having to deal with it using write_file and temp_utils from subp (which does not import any cloudinit things now). It is arguable that 'target_path' could be moved to a 'path_utils' or something, but in order to use it from subp and also from utils, we had to get it out of utils. --- cloudinit/cmd/devel/logs.py | 4 ++-- cloudinit/cmd/devel/tests/test_logs.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'cloudinit/cmd/devel') diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py index 4c086b51..51c61cca 100644 --- a/cloudinit/cmd/devel/logs.py +++ b/cloudinit/cmd/devel/logs.py @@ -12,8 +12,8 @@ import sys from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE from cloudinit.temp_utils import tempdir -from cloudinit.util import ( - ProcessExecutionError, chdir, copy, ensure_dir, subp, write_file) +from cloudinit.subp import (ProcessExecutionError, subp) +from cloudinit.util import (chdir, copy, ensure_dir, write_file) CLOUDINIT_LOGS = ['/var/log/cloud-init.log', '/var/log/cloud-init-output.log'] diff --git a/cloudinit/cmd/devel/tests/test_logs.py b/cloudinit/cmd/devel/tests/test_logs.py index d2dfa8de..ddfd58e1 100644 --- a/cloudinit/cmd/devel/tests/test_logs.py +++ b/cloudinit/cmd/devel/tests/test_logs.py @@ -8,7 +8,8 @@ from cloudinit.cmd.devel import logs from cloudinit.sources import INSTANCE_JSON_SENSITIVE_FILE from cloudinit.tests.helpers import ( FilesystemMockingTestCase, mock, wrap_and_call) -from cloudinit.util import ensure_dir, load_file, subp, write_file +from cloudinit.subp import subp +from cloudinit.util import ensure_dir, load_file, write_file @mock.patch('cloudinit.cmd.devel.logs.os.getuid') -- cgit v1.2.3 From f3bd42659efeed4b092ffcdfd5df7f24813f2d3e Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Wed, 10 Jun 2020 07:39:29 -0700 Subject: test: fix all flake8 E126 errors (#425) --- cloudinit/cmd/devel/render.py | 5 +- cloudinit/cmd/query.py | 5 +- cloudinit/distros/netbsd.py | 10 +- cloudinit/distros/openbsd.py | 7 +- cloudinit/net/__init__.py | 5 +- cloudinit/net/bsd.py | 5 +- cloudinit/net/netbsd.py | 5 +- cloudinit/net/openbsd.py | 5 +- cloudinit/net/sysconfig.py | 2 +- cloudinit/reporting/handlers.py | 14 +- cloudinit/sources/DataSourceAzure.py | 33 +++-- cloudinit/sources/helpers/tests/test_netlink.py | 165 +++++++++++++-------- cloudinit/sources/tests/test_init.py | 9 +- tests/cloud_tests/testcases/base.py | 2 +- tests/unittests/test_datasource/test_azure.py | 33 +++-- tests/unittests/test_datasource/test_scaleway.py | 42 ++++-- tests/unittests/test_distros/test_bsd_utils.py | 5 +- .../unittests/test_handler/test_handler_mounts.py | 7 +- tests/unittests/test_reporting_hyperv.py | 11 +- tests/unittests/test_sshutil.py | 18 ++- tox.ini | 3 +- 21 files changed, 238 insertions(+), 153 deletions(-) (limited to 'cloudinit/cmd/devel') diff --git a/cloudinit/cmd/devel/render.py b/cloudinit/cmd/devel/render.py index 1bc22406..1090aa16 100755 --- a/cloudinit/cmd/devel/render.py +++ b/cloudinit/cmd/devel/render.py @@ -57,8 +57,9 @@ def handle_args(name, args): paths.run_dir, INSTANCE_JSON_SENSITIVE_FILE) if not os.path.exists(instance_data_fn): LOG.warning( - 'Missing root-readable %s. Using redacted %s instead.', - instance_data_fn, redacted_data_fn) + 'Missing root-readable %s. Using redacted %s instead.', + instance_data_fn, redacted_data_fn + ) instance_data_fn = redacted_data_fn else: instance_data_fn = redacted_data_fn diff --git a/cloudinit/cmd/query.py b/cloudinit/cmd/query.py index e3db8679..0fb48ebd 100644 --- a/cloudinit/cmd/query.py +++ b/cloudinit/cmd/query.py @@ -90,8 +90,9 @@ def handle_args(name, args): instance_data_fn = sensitive_data_fn else: LOG.warning( - 'Missing root-readable %s. Using redacted %s instead.', - sensitive_data_fn, redacted_data_fn) + 'Missing root-readable %s. Using redacted %s instead.', + sensitive_data_fn, redacted_data_fn + ) instance_data_fn = redacted_data_fn else: instance_data_fn = redacted_data_fn diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py index 066737a8..f1a9b182 100644 --- a/cloudinit/distros/netbsd.py +++ b/cloudinit/distros/netbsd.py @@ -100,8 +100,9 @@ class NetBSD(cloudinit.distros.bsd.BSD): else: method = crypt.METHOD_BLOWFISH # pylint: disable=E1101 hashed_pw = crypt.crypt( - passwd, - crypt.mksalt(method)) + passwd, + crypt.mksalt(method) + ) try: subp.subp(['usermod', '-p', hashed_pw, user]) @@ -143,8 +144,9 @@ class NetBSD(cloudinit.distros.bsd.BSD): os_arch = platform.machine() e = os.environ.copy() e['PKG_PATH'] = ( - 'http://cdn.netbsd.org/pub/pkgsrc/' - 'packages/NetBSD/%s/%s/All') % (os_arch, os_release) + 'http://cdn.netbsd.org/pub/pkgsrc/' + 'packages/NetBSD/%s/%s/All' + ) % (os_arch, os_release) return e def update_package_sources(self): diff --git a/cloudinit/distros/openbsd.py b/cloudinit/distros/openbsd.py index 07c76530..720c9cf3 100644 --- a/cloudinit/distros/openbsd.py +++ b/cloudinit/distros/openbsd.py @@ -42,9 +42,10 @@ class Distro(cloudinit.distros.netbsd.NetBSD): os_arch = platform.machine() e = os.environ.copy() e['PKG_PATH'] = ( - 'ftp://ftp.openbsd.org/pub/OpenBSD/{os_release}/' - 'packages/{os_arch}/').format( - os_arch=os_arch, os_release=os_release) + 'ftp://ftp.openbsd.org/pub/OpenBSD/{os_release}/' + 'packages/{os_arch}/').format( + os_arch=os_arch, os_release=os_release + ) return e diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index a57fea0a..b40cb154 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -849,8 +849,9 @@ def get_interfaces_by_mac_on_freebsd(): def get_interfaces_by_mac_on_netbsd(): ret = {} re_field_match = ( - r"(?P\w+).*address:\s" - r"(?P([\da-f]{2}[:-]){5}([\da-f]{2})).*") + r"(?P\w+).*address:\s" + r"(?P([\da-f]{2}[:-]){5}([\da-f]{2})).*" + ) (out, _) = subp.subp(['ifconfig', '-a']) if_lines = re.sub(r'\n\s+', ' ', out).splitlines() for line in if_lines: diff --git a/cloudinit/net/bsd.py b/cloudinit/net/bsd.py index 1c355a98..e34e0454 100644 --- a/cloudinit/net/bsd.py +++ b/cloudinit/net/bsd.py @@ -66,8 +66,9 @@ class BSDRenderer(renderer.Renderer): if subnet.get('type') == 'static': if not subnet.get('netmask'): LOG.debug( - 'Skipping IP %s, because there is no netmask', - subnet.get('address')) + 'Skipping IP %s, because there is no netmask', + subnet.get('address') + ) continue LOG.debug('Configuring dev %s with %s / %s', device_name, subnet.get('address'), subnet.get('netmask')) diff --git a/cloudinit/net/netbsd.py b/cloudinit/net/netbsd.py index 30437b5f..71b38ee6 100644 --- a/cloudinit/net/netbsd.py +++ b/cloudinit/net/netbsd.py @@ -17,8 +17,9 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): if self.dhcp_interfaces(): self.set_rc_config_value('dhcpcd', 'YES') self.set_rc_config_value( - 'dhcpcd_flags', - ' '.join(self.dhcp_interfaces())) + 'dhcpcd_flags', + ' '.join(self.dhcp_interfaces()) + ) for device_name, v in self.interface_configurations.items(): if isinstance(v, dict): self.set_rc_config_value( diff --git a/cloudinit/net/openbsd.py b/cloudinit/net/openbsd.py index 489ea48b..166d77e6 100644 --- a/cloudinit/net/openbsd.py +++ b/cloudinit/net/openbsd.py @@ -19,8 +19,9 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): elif isinstance(v, dict): try: content = "inet {address} {netmask}\n".format( - address=v['address'], - netmask=v['netmask']) + address=v['address'], + netmask=v['netmask'] + ) except KeyError: LOG.error( "Invalid static configuration for %s", diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index f36c300f..0a5d481d 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -505,7 +505,7 @@ class Renderer(renderer.Renderer): iface_cfg['IPADDR6_%d' % ipv6_index] = ipv6_cidr else: iface_cfg['IPV6ADDR_SECONDARIES'] += \ - " " + ipv6_cidr + " " + ipv6_cidr else: ipv4_index = ipv4_index + 1 suff = "" if ipv4_index == 0 else str(ipv4_index) diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 00e8d2e5..6b9127b6 100755 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -139,7 +139,8 @@ class HyperVKvpReportingHandler(ReportingHandler): self.event_key_prefix = u"{0}|{1}".format(self.EVENT_PREFIX, self.incarnation_no) self.publish_thread = threading.Thread( - target=self._publish_event_routine) + target=self._publish_event_routine + ) self.publish_thread.daemon = True self.publish_thread.start() @@ -202,10 +203,15 @@ class HyperVKvpReportingHandler(ReportingHandler): uuid.uuid4()) def _encode_kvp_item(self, key, value): - data = (struct.pack("%ds%ds" % ( + data = struct.pack( + "%ds%ds" + % ( self.HV_KVP_EXCHANGE_MAX_KEY_SIZE, - self.HV_KVP_EXCHANGE_MAX_VALUE_SIZE), - key.encode('utf-8'), value.encode('utf-8'))) + self.HV_KVP_EXCHANGE_MAX_VALUE_SIZE, + ), + key.encode("utf-8"), + value.encode("utf-8"), + ) return data def _decode_kvp_item(self, record_data): diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 89312b9e..6d569057 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -523,8 +523,9 @@ class DataSourceAzure(sources.DataSource): try: crawled_data = util.log_time( - logfunc=LOG.debug, msg='Crawl of metadata service', - func=self.crawl_metadata) + logfunc=LOG.debug, msg='Crawl of metadata service', + func=self.crawl_metadata + ) except sources.InvalidMetaDataException as e: LOG.warning('Could not crawl Azure metadata: %s', e) return False @@ -893,9 +894,10 @@ def can_dev_be_reformatted(devpath, preserve_ntfs): (cand_part, cand_path, devpath)) with events.ReportEventStack( - name="mount-ntfs-and-count", - description="mount-ntfs-and-count", - parent=azure_ds_reporter) as evt: + name="mount-ntfs-and-count", + description="mount-ntfs-and-count", + parent=azure_ds_reporter + ) as evt: try: file_count = util.mount_cb(cand_path, count_files, mtype="ntfs", update_env_for_mount={'LANG': 'C'}) @@ -924,9 +926,10 @@ def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, maxwait=120, # wait for ephemeral disk to come up naplen = .2 with events.ReportEventStack( - name="wait-for-ephemeral-disk", - description="wait for ephemeral disk", - parent=azure_ds_reporter): + name="wait-for-ephemeral-disk", + description="wait for ephemeral disk", + parent=azure_ds_reporter + ): missing = util.wait_for_files([devpath], maxwait=maxwait, naplen=naplen, @@ -1334,9 +1337,10 @@ def parse_network_config(imds_metadata): @return: Dictionary containing network version 2 standard configuration. """ with events.ReportEventStack( - name="parse_network_config", - description="", - parent=azure_ds_reporter) as evt: + name="parse_network_config", + description="", + parent=azure_ds_reporter + ) as evt: if imds_metadata != sources.UNSET and imds_metadata: netconfig = {'version': 2, 'ethernets': {}} LOG.debug('Azure: generating network configuration from IMDS') @@ -1480,9 +1484,10 @@ def maybe_remove_ubuntu_network_config_scripts(paths=None): def _is_platform_viable(seed_dir): with events.ReportEventStack( - name="check-platform-viability", - description="found azure asset tag", - parent=azure_ds_reporter) as evt: + name="check-platform-viability", + description="found azure asset tag", + parent=azure_ds_reporter + ) as evt: """Check platform environment to report if this datasource may run.""" asset_tag = util.read_dmi_data('chassis-asset-tag') diff --git a/cloudinit/sources/helpers/tests/test_netlink.py b/cloudinit/sources/helpers/tests/test_netlink.py index 58c3adc6..10760bd6 100644 --- a/cloudinit/sources/helpers/tests/test_netlink.py +++ b/cloudinit/sources/helpers/tests/test_netlink.py @@ -180,17 +180,22 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): other_ifname = "eth1" expected_ifname = "eth0" data_op_down_eth1 = self._media_switch_data( - other_ifname, RTM_NEWLINK, OPER_DOWN) + other_ifname, RTM_NEWLINK, OPER_DOWN + ) data_op_up_eth1 = self._media_switch_data( - other_ifname, RTM_NEWLINK, OPER_UP) + other_ifname, RTM_NEWLINK, OPER_UP + ) data_op_down_eth0 = self._media_switch_data( - expected_ifname, RTM_NEWLINK, OPER_DOWN) + expected_ifname, RTM_NEWLINK, OPER_DOWN + ) data_op_up_eth0 = self._media_switch_data( - expected_ifname, RTM_NEWLINK, OPER_UP) - m_read_netlink_socket.side_effect = [data_op_down_eth1, - data_op_up_eth1, - data_op_down_eth0, - data_op_up_eth0] + expected_ifname, RTM_NEWLINK, OPER_UP) + m_read_netlink_socket.side_effect = [ + data_op_down_eth1, + data_op_up_eth1, + data_op_down_eth0, + data_op_up_eth0 + ] wait_for_media_disconnect_connect(m_socket, expected_ifname) self.assertIn('Ignored netlink event on interface %s' % other_ifname, self.logs.getvalue()) @@ -207,17 +212,23 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): ''' ifname = "eth0" data_getlink_down = self._media_switch_data( - ifname, RTM_GETLINK, OPER_DOWN) + ifname, RTM_GETLINK, OPER_DOWN + ) data_getlink_up = self._media_switch_data( - ifname, RTM_GETLINK, OPER_UP) + ifname, RTM_GETLINK, OPER_UP + ) data_newlink_down = self._media_switch_data( - ifname, RTM_NEWLINK, OPER_DOWN) + ifname, RTM_NEWLINK, OPER_DOWN + ) data_newlink_up = self._media_switch_data( - ifname, RTM_NEWLINK, OPER_UP) - m_read_netlink_socket.side_effect = [data_getlink_down, - data_getlink_up, - data_newlink_down, - data_newlink_up] + ifname, RTM_NEWLINK, OPER_UP + ) + m_read_netlink_socket.side_effect = [ + data_getlink_down, + data_getlink_up, + data_newlink_down, + data_newlink_up + ] wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 4) @@ -233,19 +244,25 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): ''' ifname = "eth0" data_setlink_down = self._media_switch_data( - ifname, RTM_SETLINK, OPER_DOWN) + ifname, RTM_SETLINK, OPER_DOWN + ) data_setlink_up = self._media_switch_data( - ifname, RTM_SETLINK, OPER_UP) + ifname, RTM_SETLINK, OPER_UP + ) data_newlink_down = self._media_switch_data( - ifname, RTM_NEWLINK, OPER_DOWN) + ifname, RTM_NEWLINK, OPER_DOWN + ) data_newlink_up = self._media_switch_data( - ifname, RTM_NEWLINK, OPER_UP) - m_read_netlink_socket.side_effect = [data_setlink_down, - data_setlink_up, - data_newlink_down, - data_newlink_up, - data_newlink_down, - data_newlink_up] + ifname, RTM_NEWLINK, OPER_UP + ) + m_read_netlink_socket.side_effect = [ + data_setlink_down, + data_setlink_up, + data_newlink_down, + data_newlink_up, + data_newlink_down, + data_newlink_up + ] wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 4) @@ -255,23 +272,30 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): ifname = "eth0" data_op_down = self._media_switch_data(ifname, RTM_NEWLINK, OPER_DOWN) data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP) - data_op_dormant = self._media_switch_data(ifname, RTM_NEWLINK, - OPER_DORMANT) - data_op_notpresent = self._media_switch_data(ifname, RTM_NEWLINK, - OPER_NOTPRESENT) - data_op_lowerdown = self._media_switch_data(ifname, RTM_NEWLINK, - OPER_LOWERLAYERDOWN) - data_op_testing = self._media_switch_data(ifname, RTM_NEWLINK, - OPER_TESTING) - data_op_unknown = self._media_switch_data(ifname, RTM_NEWLINK, - OPER_UNKNOWN) - m_read_netlink_socket.side_effect = [data_op_up, data_op_up, - data_op_dormant, data_op_up, - data_op_notpresent, data_op_up, - data_op_lowerdown, data_op_up, - data_op_testing, data_op_up, - data_op_unknown, data_op_up, - data_op_down, data_op_up] + data_op_dormant = self._media_switch_data( + ifname, RTM_NEWLINK, OPER_DORMANT + ) + data_op_notpresent = self._media_switch_data( + ifname, RTM_NEWLINK, OPER_NOTPRESENT + ) + data_op_lowerdown = self._media_switch_data( + ifname, RTM_NEWLINK, OPER_LOWERLAYERDOWN + ) + data_op_testing = self._media_switch_data( + ifname, RTM_NEWLINK, OPER_TESTING + ) + data_op_unknown = self._media_switch_data( + ifname, RTM_NEWLINK, OPER_UNKNOWN + ) + m_read_netlink_socket.side_effect = [ + data_op_up, data_op_up, + data_op_dormant, data_op_up, + data_op_notpresent, data_op_up, + data_op_lowerdown, data_op_up, + data_op_testing, data_op_up, + data_op_unknown, data_op_up, + data_op_down, data_op_up + ] wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 14) @@ -281,12 +305,14 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): ifname = "eth0" data_op_down = self._media_switch_data(ifname, RTM_NEWLINK, OPER_DOWN) data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP) - data_op_dormant = self._media_switch_data(ifname, RTM_NEWLINK, - OPER_DORMANT) - data_op_unknown = self._media_switch_data(ifname, RTM_NEWLINK, - OPER_UNKNOWN) - m_read_netlink_socket.side_effect = [data_op_down, data_op_dormant, - data_op_unknown, data_op_up] + data_op_dormant = self._media_switch_data( + ifname, RTM_NEWLINK, OPER_DORMANT) + data_op_unknown = self._media_switch_data( + ifname, RTM_NEWLINK, OPER_UNKNOWN) + m_read_netlink_socket.side_effect = [ + data_op_down, data_op_dormant, + data_op_unknown, data_op_up + ] wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 4) @@ -300,9 +326,11 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): data_op_down = self._media_switch_data(ifname, RTM_NEWLINK, OPER_DOWN) data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP) data_op_invalid = self._media_switch_data(ifname, RTM_NEWLINK, 7) - m_read_netlink_socket.side_effect = [data_op_invalid, data_op_up, - data_op_down, data_op_invalid, - data_op_up] + m_read_netlink_socket.side_effect = [ + data_op_invalid, data_op_up, + data_op_down, data_op_invalid, + data_op_up + ] wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 5) @@ -333,8 +361,9 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): data_invalid2 = self._media_switch_data(ifname, RTM_NEWLINK, None) data_op_down = self._media_switch_data(ifname, RTM_NEWLINK, OPER_DOWN) data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP) - m_read_netlink_socket.side_effect = [data_invalid1, data_invalid2, - data_op_down, data_op_up] + m_read_netlink_socket.side_effect = [ + data_invalid1, data_invalid2, data_op_down, data_op_up + ] wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 4) @@ -344,11 +373,15 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): bytes = ifname.encode("utf-8") data = bytearray(96) struct.pack_into("=LHHLL", data, 0, 48, RTM_NEWLINK, 0, 0, 0) - struct.pack_into("HH4sHHc", data, RTATTR_START_OFFSET, 8, 3, - bytes, 5, 16, int_to_bytes(OPER_DOWN)) + struct.pack_into( + "HH4sHHc", data, RTATTR_START_OFFSET, 8, 3, + bytes, 5, 16, int_to_bytes(OPER_DOWN) + ) struct.pack_into("=LHHLL", data, 48, 48, RTM_NEWLINK, 0, 0, 0) - struct.pack_into("HH4sHHc", data, 48 + RTATTR_START_OFFSET, 8, - 3, bytes, 5, 16, int_to_bytes(OPER_UP)) + struct.pack_into( + "HH4sHHc", data, 48 + RTATTR_START_OFFSET, 8, + 3, bytes, 5, 16, int_to_bytes(OPER_UP) + ) m_read_netlink_socket.return_value = data wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 1) @@ -360,14 +393,18 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): data1 = bytearray(112) data2 = bytearray(32) struct.pack_into("=LHHLL", data1, 0, 48, RTM_NEWLINK, 0, 0, 0) - struct.pack_into("HH4sHHc", data1, RTATTR_START_OFFSET, 8, 3, - bytes, 5, 16, int_to_bytes(OPER_DOWN)) + struct.pack_into( + "HH4sHHc", data1, RTATTR_START_OFFSET, 8, 3, + bytes, 5, 16, int_to_bytes(OPER_DOWN) + ) struct.pack_into("=LHHLL", data1, 48, 48, RTM_NEWLINK, 0, 0, 0) - struct.pack_into("HH4sHHc", data1, 80, 8, 3, bytes, 5, 16, - int_to_bytes(OPER_DOWN)) + struct.pack_into( + "HH4sHHc", data1, 80, 8, 3, bytes, 5, 16, int_to_bytes(OPER_DOWN) + ) struct.pack_into("=LHHLL", data1, 96, 48, RTM_NEWLINK, 0, 0, 0) - struct.pack_into("HH4sHHc", data2, 16, 8, 3, bytes, 5, 16, - int_to_bytes(OPER_UP)) + struct.pack_into( + "HH4sHHc", data2, 16, 8, 3, bytes, 5, 16, int_to_bytes(OPER_UP) + ) m_read_netlink_socket.side_effect = [data1, data2] wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 2) diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py index 5b6f1b3f..1420a988 100644 --- a/cloudinit/sources/tests/test_init.py +++ b/cloudinit/sources/tests/test_init.py @@ -436,10 +436,11 @@ class TestDataSource(CiTestCase): expected = { 'base64_encoded_keys': [], 'merged_cfg': { - '_doc': ( - 'Merged cloud-init system config from ' - '/etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/'), - 'datasource': {'_undef': {'key1': False}}}, + '_doc': ( + 'Merged cloud-init system config from ' + '/etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/' + ), + 'datasource': {'_undef': {'key1': False}}}, 'sensitive_keys': [ 'ds/meta_data/some/security-credentials', 'merged_cfg'], 'sys_info': sys_info, diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 68d59111..2e7c6686 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -321,7 +321,7 @@ class CloudTestCase(unittest.TestCase): "Unexpected sys_info dist value") self.assertEqual(self.os_name, v1_data['distro_release']) self.assertEqual( - str(self.os_cfg['version']), v1_data['distro_version']) + str(self.os_cfg['version']), v1_data['distro_version']) self.assertEqual('x86_64', v1_data['machine']) self.assertIsNotNone( re.match(r'3.\d\.\d', v1_data['python_version']), diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 05552a1e..a99cbd41 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -683,15 +683,17 @@ scbus-1 on xpt0 bus 0 'cloudinit.sources.DataSourceAzure.DataSourceAzure._report_ready') @mock.patch('cloudinit.sources.DataSourceAzure.DataSourceAzure._poll_imds') def test_crawl_metadata_on_reprovision_reports_ready( - self, poll_imds_func, - report_ready_func, - m_write, m_dhcp): + self, poll_imds_func, report_ready_func, m_write, m_dhcp + ): """If reprovisioning, report ready at the end""" ovfenv = construct_valid_ovf_env( - platform_settings={"PreprovisionedVm": "True"}) + platform_settings={"PreprovisionedVm": "True"} + ) - data = {'ovfcontent': ovfenv, - 'sys_cfg': {}} + data = { + 'ovfcontent': ovfenv, + 'sys_cfg': {} + } dsrc = self._get_ds(data) poll_imds_func.return_value = ovfenv dsrc.crawl_metadata() @@ -706,15 +708,18 @@ scbus-1 on xpt0 bus 0 @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') @mock.patch('cloudinit.sources.DataSourceAzure.readurl') def test_crawl_metadata_on_reprovision_reports_ready_using_lease( - self, m_readurl, m_dhcp, - m_net, report_ready_func, - m_media_switch, m_write): + self, m_readurl, m_dhcp, m_net, report_ready_func, + m_media_switch, m_write + ): """If reprovisioning, report ready using the obtained lease""" ovfenv = construct_valid_ovf_env( - platform_settings={"PreprovisionedVm": "True"}) + platform_settings={"PreprovisionedVm": "True"} + ) - data = {'ovfcontent': ovfenv, - 'sys_cfg': {}} + data = { + 'ovfcontent': ovfenv, + 'sys_cfg': {} + } dsrc = self._get_ds(data) lease = { @@ -1955,8 +1960,8 @@ class TestPreprovisioningPollIMDS(CiTestCase): response = requests.Response() response.status_code = 404 if self.tries == 2 else 410 raise requests.exceptions.HTTPError( - "fake {}".format(response.status_code), - response=response) + "fake {}".format(response.status_code), response=response + ) # Third try should succeed and stop retries or redhcp return mock.MagicMock(status_code=200, text="good", content="good") diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py index 15441454..9d82bda9 100644 --- a/tests/unittests/test_datasource/test_scaleway.py +++ b/tests/unittests/test_datasource/test_scaleway.py @@ -353,12 +353,16 @@ class TestDataSourceScaleway(HttprettyTestCase): self.datasource.metadata['ipv6'] = None netcfg = self.datasource.network_config - resp = {'version': 1, - 'config': [{ - 'type': 'physical', - 'name': 'ens2', - 'subnets': [{'type': 'dhcp4'}]}] + resp = { + 'version': 1, + 'config': [ + { + 'type': 'physical', + 'name': 'ens2', + 'subnets': [{'type': 'dhcp4'}] } + ] + } self.assertEqual(netcfg, resp) @mock.patch('cloudinit.sources.DataSourceScaleway.net.find_fallback_nic') @@ -424,12 +428,16 @@ class TestDataSourceScaleway(HttprettyTestCase): self.datasource.metadata['ipv6'] = None self.datasource._network_config = sources.UNSET - resp = {'version': 1, - 'config': [{ - 'type': 'physical', - 'name': 'ens2', - 'subnets': [{'type': 'dhcp4'}]}] + resp = { + 'version': 1, + 'config': [ + { + 'type': 'physical', + 'name': 'ens2', + 'subnets': [{'type': 'dhcp4'}] } + ] + } netcfg = self.datasource.network_config self.assertEqual(netcfg, resp) @@ -448,12 +456,16 @@ class TestDataSourceScaleway(HttprettyTestCase): self.datasource.metadata['ipv6'] = None self.datasource._network_config = None - resp = {'version': 1, - 'config': [{ - 'type': 'physical', - 'name': 'ens2', - 'subnets': [{'type': 'dhcp4'}]}] + resp = { + 'version': 1, + 'config': [ + { + 'type': 'physical', + 'name': 'ens2', + 'subnets': [{'type': 'dhcp4'}] } + ] + } netcfg = self.datasource.network_config self.assertEqual(netcfg, resp) diff --git a/tests/unittests/test_distros/test_bsd_utils.py b/tests/unittests/test_distros/test_bsd_utils.py index b38e4af5..3a68f2a9 100644 --- a/tests/unittests/test_distros/test_bsd_utils.py +++ b/tests/unittests/test_distros/test_bsd_utils.py @@ -62,5 +62,6 @@ class TestBsdUtils(CiTestCase): self.load_file.return_value = RC_FILE.format(hostname='foo') bsd_utils.set_rc_config_value('hostname', 'bar') self.write_file.assert_called_with( - '/etc/rc.conf', - RC_FILE.format(hostname='bar')) + '/etc/rc.conf', + RC_FILE.format(hostname='bar') + ) diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index 80c53c83..b643e3ae 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -260,8 +260,11 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase): '/dev/vdb /mnt auto defaults,noexec,comment=cloudconfig 0 2\n' ) fstab_expected_content = fstab_original_content - cc = {'mounts': [ - ['/dev/vdb', '/mnt', 'auto', 'defaults,noexec']]} + cc = { + 'mounts': [ + ['/dev/vdb', '/mnt', 'auto', 'defaults,noexec'] + ] + } with open(cc_mounts.FSTAB_PATH, 'w') as fd: fd.write(fstab_original_content) with open(cc_mounts.FSTAB_PATH, 'r') as fd: diff --git a/tests/unittests/test_reporting_hyperv.py b/tests/unittests/test_reporting_hyperv.py index fa8f8859..b60a66ab 100644 --- a/tests/unittests/test_reporting_hyperv.py +++ b/tests/unittests/test_reporting_hyperv.py @@ -93,10 +93,15 @@ class TextKvpReporter(CiTestCase): def test_not_truncate_kvp_file_modified_after_boot(self): with open(self.tmp_file_path, "wb+") as f: kvp = {'key': 'key1', 'value': 'value1'} - data = (struct.pack("%ds%ds" % ( + data = struct.pack( + "%ds%ds" + % ( HyperVKvpReportingHandler.HV_KVP_EXCHANGE_MAX_KEY_SIZE, - HyperVKvpReportingHandler.HV_KVP_EXCHANGE_MAX_VALUE_SIZE), - kvp['key'].encode('utf-8'), kvp['value'].encode('utf-8'))) + HyperVKvpReportingHandler.HV_KVP_EXCHANGE_MAX_VALUE_SIZE, + ), + kvp["key"].encode("utf-8"), + kvp["value"].encode("utf-8"), + ) f.write(data) cur_time = time.time() os.utime(self.tmp_file_path, (cur_time, cur_time)) diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index b4767f0c..d15fc60b 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -374,13 +374,13 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase): sshd_config = self.tmp_path('sshd_config') util.write_file( - sshd_config, - "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys)) + sshd_config, + "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys) + ) (auth_key_fn, auth_key_entries) = ssh_util.extract_authorized_keys( - fpw.pw_name, sshd_config) - content = ssh_util.update_authorized_keys( - auth_key_entries, []) + fpw.pw_name, sshd_config) + content = ssh_util.update_authorized_keys(auth_key_entries, []) self.assertEqual("%s/.ssh/authorized_keys" % fpw.pw_dir, auth_key_fn) self.assertTrue(VALID_CONTENT['rsa'] in content) @@ -398,11 +398,13 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase): sshd_config = self.tmp_path('sshd_config') util.write_file( - sshd_config, - "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys)) + sshd_config, + "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys) + ) (auth_key_fn, auth_key_entries) = ssh_util.extract_authorized_keys( - fpw.pw_name, sshd_config) + fpw.pw_name, sshd_config + ) content = ssh_util.update_authorized_keys(auth_key_entries, []) self.assertEqual("%s/.ssh/authorized_keys" % fpw.pw_dir, auth_key_fn) diff --git a/tox.ini b/tox.ini index e04c7791..ebcebc41 100644 --- a/tox.ini +++ b/tox.ini @@ -43,11 +43,10 @@ basepython = python2.7 deps = -r{toxinidir}/test-requirements.txt [flake8] -# E126: continuation line over-indented for hanging indent # E226: missing whitespace around arithmetic operator # W503: line break before binary operator # W504: line break after binary operator -ignore=E126,E226,W503,W504 +ignore=E226,W503,W504 exclude = .venv,.tox,dist,doc,*egg,.git,build,tools per-file-ignores = cloudinit/cmd/main.py:E402 -- cgit v1.2.3 From a4b6b96f30bdd994ab535b222cf4b4bf09f20668 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Mon, 17 Aug 2020 11:06:20 -0500 Subject: cli: add devel make-mime subcommand (#518) * cli: add devel make-mime subcommand Cloud-init documents an in-source-tree tool, make-mime.py used to help users create multi-part mime user-data. This tool is not shipped in the cloud-init install and unavailable at runtime. This patch takes tools/make-mime.py and makes the functionality available via the devel subcommand. The primary interface of --attach file:content-type is still present. The cli now adds: -l, --list-types Print out a list of supported content-types -f, --force Ignore errors for unsupported content-types The tool will now raise a RunTime error if the supplied content-type is not supported (or more likely a typo: x-shell-script vs. x-shellscript) * make-mime: write to stderr and exit 1 instead of raising RuntimeError * Update example to match docs * Update docs for make-mime subcommand * Remove tools/make-mime.py; replaced by cloud-init devel make-mime Co-authored-by: Rick Harding --- cloudinit/cmd/devel/make_mime.py | 112 +++++++++++++++++++++++++++++++++++++++ cloudinit/cmd/devel/parser.py | 5 +- doc/rtd/topics/format.rst | 44 +++++++-------- tools/make-mime.py | 62 ---------------------- 4 files changed, 139 insertions(+), 84 deletions(-) create mode 100755 cloudinit/cmd/devel/make_mime.py delete mode 100755 tools/make-mime.py (limited to 'cloudinit/cmd/devel') diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py new file mode 100755 index 00000000..77e10540 --- /dev/null +++ b/cloudinit/cmd/devel/make_mime.py @@ -0,0 +1,112 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Generate multi-part mime messages for user-data """ + +import argparse +import sys +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +from cloudinit import log +from cloudinit.handlers import INCLUSION_TYPES_MAP +from . import addLogHandlerCLI + +NAME = 'make-mime' +LOG = log.getLogger(NAME) +EPILOG = ("Example: make-mime -a config.yaml:cloud-config " + "-a script.sh:x-shellscript > user-data") + + +def file_content_type(text): + """ Return file content type by reading the first line of the input. """ + try: + filename, content_type = text.split(":", 1) + return (open(filename, 'r'), filename, content_type.strip()) + except ValueError: + raise argparse.ArgumentError(text, "Invalid value for %r" % (text)) + + +def get_parser(parser=None): + """Build or extend and arg parser for make-mime utility. + + @param parser: Optional existing ArgumentParser instance representing the + subcommand which will be extended to support the args of this utility. + + @returns: ArgumentParser with proper argument configuration. + """ + if not parser: + parser = argparse.ArgumentParser() + # update the parser's doc and add an epilog to show an example + parser.description = __doc__ + parser.epilog = EPILOG + parser.add_argument("-a", "--attach", dest="files", type=file_content_type, + action='append', default=[], + metavar=":", + help=("attach the given file as the specified " + "content-type")) + parser.add_argument('-l', '--list-types', action='store_true', + default=False, + help='List support cloud-init content types.') + parser.add_argument('-f', '--force', action='store_true', + default=False, + help='Ignore unknown content-type warnings') + return parser + + +def get_content_types(strip_prefix=False): + """ Return a list of cloud-init supported content types. Optionally + strip out the leading 'text/' of the type if strip_prefix=True. + """ + return sorted([ctype.replace("text/", "") if strip_prefix else ctype + for ctype in INCLUSION_TYPES_MAP.values()]) + + +def handle_args(name, args): + """Create a multi-part MIME archive for use as user-data. Optionally + print out the list of supported content types of cloud-init. + + Also setup CLI log handlers to report to stderr since this is a development + utility which should be run by a human on the CLI. + + @return 0 on success, 1 on failure. + """ + addLogHandlerCLI(LOG, log.DEBUG if args.debug else log.WARNING) + if args.list_types: + print("\n".join(get_content_types(strip_prefix=True))) + return 0 + + sub_messages = [] + errors = [] + for i, (fh, filename, format_type) in enumerate(args.files): + contents = fh.read() + sub_message = MIMEText(contents, format_type, sys.getdefaultencoding()) + sub_message.add_header('Content-Disposition', + 'attachment; filename="%s"' % (filename)) + content_type = sub_message.get_content_type().lower() + if content_type not in get_content_types(): + level = "WARNING" if args.force else "ERROR" + msg = (level + ": content type %r for attachment %s " + "may be incorrect!") % (content_type, i + 1) + sys.stderr.write(msg + '\n') + errors.append(msg) + sub_messages.append(sub_message) + if len(errors) and not args.force: + sys.stderr.write("Invalid content-types, override with --force\n") + return 1 + combined_message = MIMEMultipart() + for msg in sub_messages: + combined_message.attach(msg) + print(combined_message) + return 0 + + +def main(): + args = get_parser().parse_args() + return(handle_args(NAME, args)) + + +if __name__ == '__main__': + sys.exit(main()) + + +# vi: ts=4 expandtab diff --git a/cloudinit/cmd/devel/parser.py b/cloudinit/cmd/devel/parser.py index 99a234ce..1a3c46a4 100644 --- a/cloudinit/cmd/devel/parser.py +++ b/cloudinit/cmd/devel/parser.py @@ -9,6 +9,7 @@ from cloudinit.config import schema from . import net_convert from . import render +from . import make_mime def get_parser(parser=None): @@ -25,7 +26,9 @@ def get_parser(parser=None): (net_convert.NAME, net_convert.__doc__, net_convert.get_parser, net_convert.handle_args), (render.NAME, render.__doc__, - render.get_parser, render.handle_args) + render.get_parser, render.handle_args), + (make_mime.NAME, make_mime.__doc__, + make_mime.get_parser, make_mime.handle_args), ] for (subcmd, helpmsg, get_parser, handler) in subcmds: parser = subparsers.add_parser(subcmd, help=helpmsg) diff --git a/doc/rtd/topics/format.rst b/doc/rtd/topics/format.rst index 1d0a7097..d03e4caf 100644 --- a/doc/rtd/topics/format.rst +++ b/doc/rtd/topics/format.rst @@ -23,33 +23,35 @@ Using a mime-multi part file, the user can specify more than one type of data. For example, both a user data script and a cloud-config type could be specified. -Supported content-types: +Supported content-types are listed from the cloud-init subcommand make-mime:: -- text/cloud-boothook -- text/cloud-config -- text/cloud-config-archive -- text/jinja2 -- text/part-handler -- text/upstart-job -- text/x-include-once-url -- text/x-include-url -- text/x-shellscript + % cloud-init devel make-mime --list-types + cloud-boothook + cloud-config + cloud-config-archive + cloud-config-jsonp + jinja2 + part-handler + upstart-job + x-include-once-url + x-include-url + x-shellscript -Helper script to generate mime messages ---------------------------------------- -The cloud-init codebase includes a helper script to generate MIME multi-part -files: `make-mime.py`_. +Helper subcommand to generate mime messages +------------------------------------------- -``make-mime.py`` takes pairs of (filename, "text/" mime subtype) separated by -a colon (e.g. ``config.yaml:cloud-config``) and emits a MIME multipart -message to stdout. An example invocation, assuming you have your cloud config -in ``config.yaml`` and a shell script in ``script.sh`` and want to store the -multipart message in ``user-data``:: +The cloud-init subcommand can generate MIME multi-part files: `make-mime`_. - ./tools/make-mime.py -a config.yaml:cloud-config -a script.sh:x-shellscript > user-data +``make-mime`` subcommand takes pairs of (filename, "text/" mime subtype) +separated by a colon (e.g. ``config.yaml:cloud-config``) and emits a MIME +multipart message to stdout. An example invocation, assuming you have your +cloud config in ``config.yaml`` and a shell script in ``script.sh`` and want +to store the multipart message in ``user-data``:: -.. _make-mime.py: https://github.com/canonical/cloud-init/blob/master/tools/make-mime.py + % cloud-init devel make-mime -a config.yaml:cloud-config -a script.sh:x-shellscript > user-data + +.. _make-mime: https://github.com/canonical/cloud-init/blob/master/cloudinit/cmd/devel/make_mime.py User-Data Script diff --git a/tools/make-mime.py b/tools/make-mime.py deleted file mode 100755 index e0022302..00000000 --- a/tools/make-mime.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import sys - -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText - -KNOWN_CONTENT_TYPES = [ - 'text/x-include-once-url', - 'text/x-include-url', - 'text/cloud-config-archive', - 'text/upstart-job', - 'text/cloud-config', - 'text/part-handler', - 'text/x-shellscript', - 'text/cloud-boothook', -] - - -def file_content_type(text): - try: - filename, content_type = text.split(":", 1) - return (open(filename, 'r'), filename, content_type.strip()) - except ValueError: - raise argparse.ArgumentError(text, "Invalid value for %r" % (text)) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("-a", "--attach", - dest="files", - type=file_content_type, - action='append', - default=[], - required=True, - metavar=":", - help="attach the given file in the specified " - "content type") - args = parser.parse_args() - sub_messages = [] - for i, (fh, filename, format_type) in enumerate(args.files): - contents = fh.read() - sub_message = MIMEText(contents, format_type, sys.getdefaultencoding()) - sub_message.add_header('Content-Disposition', - 'attachment; filename="%s"' % (filename)) - content_type = sub_message.get_content_type().lower() - if content_type not in KNOWN_CONTENT_TYPES: - sys.stderr.write(("WARNING: content type %r for attachment %s " - "may be incorrect!\n") % (content_type, i + 1)) - sub_messages.append(sub_message) - combined_message = MIMEMultipart() - for msg in sub_messages: - combined_message.attach(msg) - print(combined_message) - return 0 - - -if __name__ == '__main__': - sys.exit(main()) - -# vi: ts=4 expandtab -- cgit v1.2.3 From 07104504ab5b30efd2d1f7a8c36effe18b8e5fe0 Mon Sep 17 00:00:00 2001 From: Paride Legovini Date: Tue, 25 Aug 2020 17:21:18 +0200 Subject: tox: bump the pylint version to 2.6.0 in the default run (#544) Changes: tox: bump the pylint version to 2.6.0 in the default run Fix pylint 2.6.0 W0707 warnings (raise-missing-from) --- cloudinit/analyze/show.py | 2 +- cloudinit/cmd/devel/make_mime.py | 6 ++-- cloudinit/config/cc_disk_setup.py | 36 ++++++++++++++-------- cloudinit/config/cc_growpart.py | 8 ++--- cloudinit/config/cc_power_state_change.py | 12 +++++--- cloudinit/config/cc_rsyslog.py | 6 ++-- cloudinit/config/cc_set_hostname.py | 2 +- cloudinit/config/cc_ubuntu_advantage.py | 2 +- cloudinit/config/schema.py | 2 +- cloudinit/distros/__init__.py | 5 +-- cloudinit/distros/arch.py | 4 +-- cloudinit/distros/parsers/resolv_conf.py | 7 +++-- cloudinit/gpg.py | 5 +-- cloudinit/handlers/jinja_template.py | 3 +- cloudinit/net/__init__.py | 7 +++-- cloudinit/net/cmdline.py | 4 +-- cloudinit/net/dhcp.py | 4 +-- cloudinit/net/network_state.py | 19 +++++++----- cloudinit/sources/DataSourceAzure.py | 2 +- cloudinit/sources/DataSourceEc2.py | 8 +++-- cloudinit/sources/DataSourceIBMCloud.py | 3 +- cloudinit/sources/DataSourceMAAS.py | 3 +- cloudinit/sources/DataSourceOpenNebula.py | 13 +++++--- cloudinit/sources/DataSourceOpenStack.py | 4 +-- cloudinit/sources/DataSourceSmartOS.py | 4 ++- cloudinit/sources/helpers/azure.py | 34 +++++++++++--------- cloudinit/sources/helpers/netlink.py | 2 +- cloudinit/sources/helpers/openstack.py | 34 +++++++++++--------- cloudinit/subp.py | 3 +- cloudinit/url_helper.py | 6 ++-- cloudinit/util.py | 8 ++--- tests/cloud_tests/platforms/azurecloud/instance.py | 7 +++-- tests/cloud_tests/platforms/azurecloud/platform.py | 23 +++++++++----- tests/cloud_tests/platforms/ec2/instance.py | 4 +-- tests/cloud_tests/platforms/ec2/platform.py | 18 ++++++----- tests/cloud_tests/platforms/lxd/instance.py | 3 +- tests/cloud_tests/platforms/platforms.py | 6 ++-- tests/cloud_tests/testcases/__init__.py | 6 ++-- tools/mock-meta.py | 14 +++++---- tox.ini | 2 +- 40 files changed, 205 insertions(+), 136 deletions(-) (limited to 'cloudinit/cmd/devel') diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py index 0c825b23..01a4d3e5 100644 --- a/cloudinit/analyze/show.py +++ b/cloudinit/analyze/show.py @@ -267,7 +267,7 @@ def gather_timestamps_using_systemd(): except OSError as err: raise RuntimeError('Could not determine container boot ' 'time from /proc/1/cmdline. ({})' - .format(err)) + .format(err)) from err status = CONTAINER_CODE else: status = FAIL_CODE diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index 77e10540..4e6a5778 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -22,8 +22,10 @@ def file_content_type(text): try: filename, content_type = text.split(":", 1) return (open(filename, 'r'), filename, content_type.strip()) - except ValueError: - raise argparse.ArgumentError(text, "Invalid value for %r" % (text)) + except ValueError as e: + raise argparse.ArgumentError( + text, "Invalid value for %r" % (text) + ) from e def get_parser(parser=None): diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index d957cfe3..a7bdc703 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -251,7 +251,9 @@ def enumerate_disk(device, nodeps=False): try: info, _err = subp.subp(lsblk_cmd) except Exception as e: - raise Exception("Failed during disk check for %s\n%s" % (device, e)) + raise Exception( + "Failed during disk check for %s\n%s" % (device, e) + ) from e parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0] @@ -313,7 +315,9 @@ def check_fs(device): try: out, _err = subp.subp(blkid_cmd, rcs=[0, 2]) except Exception as e: - raise Exception("Failed during disk check for %s\n%s" % (device, e)) + raise Exception( + "Failed during disk check for %s\n%s" % (device, e) + ) from e if out: if len(out.splitlines()) == 1: @@ -428,8 +432,8 @@ def get_dyn_func(*args): else: return globals()[func_name] - except KeyError: - raise Exception("No such function %s to call!" % func_name) + except KeyError as e: + raise Exception("No such function %s to call!" % func_name) from e def get_hdd_size(device): @@ -437,7 +441,7 @@ def get_hdd_size(device): size_in_bytes, _ = subp.subp([BLKDEV_CMD, '--getsize64', device]) sector_size, _ = subp.subp([BLKDEV_CMD, '--getss', device]) except Exception as e: - raise Exception("Failed to get %s size\n%s" % (device, e)) + raise Exception("Failed to get %s size\n%s" % (device, e)) from e return int(size_in_bytes) / int(sector_size) @@ -455,8 +459,9 @@ def check_partition_mbr_layout(device, layout): try: out, _err = subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: - raise Exception("Error running partition command on %s\n%s" % ( - device, e)) + raise Exception( + "Error running partition command on %s\n%s" % (device, e) + ) from e found_layout = [] for line in out.splitlines(): @@ -485,8 +490,9 @@ def check_partition_gpt_layout(device, layout): try: out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV) except Exception as e: - raise Exception("Error running partition command on %s\n%s" % ( - device, e)) + raise Exception( + "Error running partition command on %s\n%s" % (device, e) + ) from e out_lines = iter(out.splitlines()) # Skip header. Output looks like: @@ -657,8 +663,10 @@ def purge_disk(device): try: LOG.info("Purging filesystem on /dev/%s", d['name']) subp.subp(wipefs_cmd) - except Exception: - raise Exception("Failed FS purge of /dev/%s" % d['name']) + except Exception as e: + raise Exception( + "Failed FS purge of /dev/%s" % d['name'] + ) from e purge_disk_ptable(device) @@ -700,7 +708,9 @@ def exec_mkpart_mbr(device, layout): try: subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: - raise Exception("Failed to partition device %s\n%s" % (device, e)) + raise Exception( + "Failed to partition device %s\n%s" % (device, e) + ) from e read_parttbl(device) @@ -997,6 +1007,6 @@ def mkfs(fs_cfg): try: subp.subp(fs_cmd, shell=shell) except Exception as e: - raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) + raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) from e # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index c5d93f81..237c3d02 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -148,14 +148,14 @@ class ResizeGrowPart(object): if e.exit_code != 1: util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)", diskdev, partnum) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e return (before, before) try: subp.subp(["growpart", diskdev, partnum]) except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: growpart %s %s", diskdev, partnum) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e return (before, get_size(partdev)) @@ -187,14 +187,14 @@ class ResizeGpart(object): except subp.ProcessExecutionError as e: if e.exit_code != 0: util.logexc(LOG, "Failed: gpart recover %s", diskdev) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e before = get_size(partdev) try: subp.subp(["gpart", "resize", "-i", partnum, diskdev]) except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e # Since growing the FS requires a reboot, make sure we reboot # first when this module has finished. diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index ab953a0d..6fcb8a7d 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -196,10 +196,11 @@ def load_power_state(cfg, distro_name): try: delay = convert_delay(delay, fmt=fmt, scale=scale) - except ValueError: + except ValueError as e: raise TypeError( "power_state[delay] must be 'now' or '+m' (minutes)." - " found '%s'." % delay) + " found '%s'." % delay + ) from e args = command + [delay] if message: @@ -207,9 +208,10 @@ def load_power_state(cfg, distro_name): try: timeout = float(pstate.get('timeout', 30.0)) - except ValueError: - raise ValueError("failed to convert timeout '%s' to float." % - pstate['timeout']) + except ValueError as e: + raise ValueError( + "failed to convert timeout '%s' to float." % pstate['timeout'] + ) from e condition = pstate.get("condition", True) if not isinstance(condition, (str, list, bool)): diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 1354885a..2a2bc931 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -347,8 +347,10 @@ class SyslogRemotesLine(object): if self.port: try: int(self.port) - except ValueError: - raise ValueError("port '%s' is not an integer" % self.port) + except ValueError as e: + raise ValueError( + "port '%s' is not an integer" % self.port + ) from e if not self.addr: raise ValueError("address is required") diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index c13020d8..1d23d80d 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -85,7 +85,7 @@ def handle(name, cfg, cloud, log, _args): except Exception as e: msg = "Failed to set the hostname to %s (%s)" % (fqdn, hostname) util.logexc(log, msg) - raise SetHostnameError("%s: %s" % (msg, e)) + raise SetHostnameError("%s: %s" % (msg, e)) from e write_json(prev_fn, {'hostname': hostname, 'fqdn': fqdn}) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py index 35ded5db..d61dc655 100644 --- a/cloudinit/config/cc_ubuntu_advantage.py +++ b/cloudinit/config/cc_ubuntu_advantage.py @@ -115,7 +115,7 @@ def configure_ua(token=None, enable=None): msg = 'Failure attaching Ubuntu Advantage:\n{error}'.format( error=str(e)) util.logexc(LOG, msg) - raise RuntimeError(msg) + raise RuntimeError(msg) from e enable_errors = [] for service in enable: try: diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 2d8c7577..8a966aee 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -210,7 +210,7 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): error = SchemaValidationError(errors) if annotate: print(annotated_cloudconfig_file({}, content, error.schema_errors)) - raise error + raise error from e try: validate_cloudconfig_schema( cloudconfig, schema, strict=True) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index effb4276..2537608f 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -607,10 +607,11 @@ class Distro(metaclass=abc.ABCMeta): lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name]) try: cmd = next(tool for tool in lock_tools if subp.which(tool[0])) - except StopIteration: + except StopIteration as e: raise RuntimeError(( "Unable to lock user account '%s'. No tools available. " - " Tried: %s.") % (name, [c[0] for c in lock_tools])) + " Tried: %s.") % (name, [c[0] for c in lock_tools]) + ) from e try: subp.subp(cmd) except Exception as e: diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 038aa9ac..967be168 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -61,9 +61,9 @@ class Distro(distros.Distro): def _write_network_config(self, netconfig): try: return self._supported_write_network_config(netconfig) - except RendererNotFoundError: + except RendererNotFoundError as e: # Fall back to old _write_network - raise NotImplementedError + raise NotImplementedError from e def _write_network(self, settings): entries = net_util.translate_network(settings) diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py index 299d54b5..62929d03 100644 --- a/cloudinit/distros/parsers/resolv_conf.py +++ b/cloudinit/distros/parsers/resolv_conf.py @@ -150,9 +150,10 @@ class ResolvConf(object): tail = '' try: (cfg_opt, cfg_values) = head.split(None, 1) - except (IndexError, ValueError): - raise IOError("Incorrectly formatted resolv.conf line %s" - % (i + 1)) + except (IndexError, ValueError) as e: + raise IOError( + "Incorrectly formatted resolv.conf line %s" % (i + 1) + ) from e if cfg_opt not in ['nameserver', 'domain', 'search', 'sortlist', 'options']: raise IOError("Unexpected resolv.conf option %s" % (cfg_opt)) diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py index 72b5ac59..be0ca0ea 100644 --- a/cloudinit/gpg.py +++ b/cloudinit/gpg.py @@ -63,10 +63,11 @@ def recv_key(key, keyserver, retries=(1, 1)): "Import failed with exit code %d, will try again in %ss", error.exit_code, naplen) time.sleep(naplen) - except StopIteration: + except StopIteration as e: raise ValueError( ("Failed to import key '%s' from keyserver '%s' " - "after %d tries: %s") % (key, keyserver, trynum, error)) + "after %d tries: %s") % (key, keyserver, trynum, error) + ) from e def delete_key(key): diff --git a/cloudinit/handlers/jinja_template.py b/cloudinit/handlers/jinja_template.py index ce3accf6..aadfbf86 100644 --- a/cloudinit/handlers/jinja_template.py +++ b/cloudinit/handlers/jinja_template.py @@ -83,7 +83,8 @@ def render_jinja_payload_from_file( if e.errno == EACCES: raise RuntimeError( 'Cannot render jinja template vars. No read permission on' - " '%s'. Try sudo" % instance_data_file) + " '%s'. Try sudo" % instance_data_file + ) from e rendered_payload = render_jinja_payload( payload, payload_fn, instance_data, debug) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 322af77b..e233149a 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -506,7 +506,9 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): try: _rename_interfaces(extract_physdevs(netcfg)) except RuntimeError as e: - raise RuntimeError('Failed to apply network config names: %s' % e) + raise RuntimeError( + 'Failed to apply network config names: %s' % e + ) from e def interface_has_own_mac(ifname, strict=False): @@ -965,7 +967,8 @@ class EphemeralIPv4Network(object): self.prefix = mask_to_net_prefix(prefix_or_mask) except ValueError as e: raise ValueError( - 'Cannot setup network: {0}'.format(e)) + 'Cannot setup network: {0}'.format(e) + ) from e self.connectivity_url = connectivity_url self.interface = interface diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 7ca7262b..cc8dc17b 100755 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -112,8 +112,8 @@ def _klibc_to_config_entry(content, mac_addrs=None): data = util.load_shell_content(content) try: name = data['DEVICE'] if 'DEVICE' in data else data['DEVICE6'] - except KeyError: - raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data") + except KeyError as e: + raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data") from e # ipconfig on precise does not write PROTO # IPv6 config gives us IPV6PROTO, not PROTO. diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 9230cf7a..4394c68b 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -82,8 +82,8 @@ class EphemeralDHCPv4(object): try: leases = maybe_perform_dhcp_discovery( self.iface, self.dhcp_log_func) - except InvalidDHCPLeaseFileError: - raise NoDHCPLeaseError() + except InvalidDHCPLeaseFileError as e: + raise NoDHCPLeaseError() from e if not leases: raise NoDHCPLeaseError() self.lease = leases[-1] diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 7bfe8be0..b2f7d31e 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -297,9 +297,10 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): command_type = command['type'] try: handler = self.command_handlers[command_type] - except KeyError: - raise RuntimeError("No handler found for" - " command '%s'" % command_type) + except KeyError as e: + raise RuntimeError( + "No handler found for command '%s'" % command_type + ) from e try: handler(self, command) except InvalidCommand: @@ -316,9 +317,10 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): continue try: handler = self.command_handlers[command_type] - except KeyError: - raise RuntimeError("No handler found for" - " command '%s'" % command_type) + except KeyError as e: + raise RuntimeError( + "No handler found for command '%s'" % command_type + ) from e try: handler(self, command) self._v2_common(command) @@ -914,9 +916,10 @@ def _normalize_route(route): if metric: try: normal_route['metric'] = int(metric) - except ValueError: + except ValueError as e: raise TypeError( - 'Route config metric {} is not an integer'.format(metric)) + 'Route config metric {} is not an integer'.format(metric) + ) from e return normal_route diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 86cc7c28..f3c6452b 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -1147,7 +1147,7 @@ def read_azure_ovf(contents): except Exception as e: error_str = "Invalid ovf-env.xml: %s" % e report_diagnostic_event(error_str) - raise BrokenAzureDataSource(error_str) + raise BrokenAzureDataSource(error_str) from e results = find_child(dom.documentElement, lambda n: n.localName == "ProvisioningSection") diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 355b4e2f..1d09c12a 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -617,9 +617,11 @@ def parse_strict_mode(cfgval): if sleep: try: sleep = int(sleep) - except ValueError: - raise ValueError("Invalid sleep '%s' in strict_id setting '%s': " - "not an integer" % (sleep, cfgval)) + except ValueError as e: + raise ValueError( + "Invalid sleep '%s' in strict_id setting '%s': not an integer" + % (sleep, cfgval) + ) from e else: sleep = None diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py index d2aa9a58..8d196185 100644 --- a/cloudinit/sources/DataSourceIBMCloud.py +++ b/cloudinit/sources/DataSourceIBMCloud.py @@ -303,7 +303,8 @@ def read_md(): except sources.BrokenMetadata as e: raise RuntimeError( "Failed reading IBM config disk (platform=%s path=%s): %s" % - (platform, path, e)) + (platform, path, e) + ) from e ret.update(results) return ret diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index c80f70c2..9156925f 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -226,7 +226,8 @@ def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None, except url_helper.UrlError as e: if e.code == 404 and not optional: raise MAASSeedDirMalformed( - "Missing required %s: %s" % (path, e)) + "Missing required %s: %s" % (path, e) + ) from e elif e.code != 404: raise e diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 12b1f94f..45481938 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -399,18 +399,23 @@ def read_context_disk_dir(source_dir, distro, asuser=None): if asuser is not None: try: pwd.getpwnam(asuser) - except KeyError: + except KeyError as e: raise BrokenContextDiskDir( "configured user '{user}' does not exist".format( - user=asuser)) + user=asuser) + ) from e try: path = os.path.join(source_dir, 'context.sh') content = util.load_file(path) context = parse_shell_config(content, asuser=asuser) except subp.ProcessExecutionError as e: - raise BrokenContextDiskDir("Error processing context.sh: %s" % (e)) + raise BrokenContextDiskDir( + "Error processing context.sh: %s" % (e) + ) from e except IOError as e: - raise NonContextDiskDir("Error reading context.sh: %s" % (e)) + raise NonContextDiskDir( + "Error reading context.sh: %s" % (e) + ) from e else: raise NonContextDiskDir("Missing context.sh") diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index bf539091..d4b43f44 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -194,10 +194,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'timeout': url_params.timeout_seconds}) except openstack.NonReadable as e: raise sources.InvalidMetaDataException(str(e)) - except (openstack.BrokenMetadata, IOError): + except (openstack.BrokenMetadata, IOError) as e: msg = 'Broken metadata address {addr}'.format( addr=self.metadata_address) - raise sources.InvalidMetaDataException(msg) + raise sources.InvalidMetaDataException(msg) from e return result diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 843b3a2a..f1f903bc 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -413,7 +413,9 @@ class JoyentMetadataClient(object): response.append(byte) except OSError as exc: if exc.errno == errno.EAGAIN: - raise JoyentMetadataTimeoutException(msg % as_ascii()) + raise JoyentMetadataTimeoutException( + msg % as_ascii() + ) from exc raise def _write(self, msg): diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 6156c75b..b968a96f 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -98,8 +98,10 @@ def get_boot_telemetry(): LOG.debug("Collecting boot telemetry") try: kernel_start = float(time.time()) - float(util.uptime()) - except ValueError: - raise RuntimeError("Failed to determine kernel start timestamp") + except ValueError as e: + raise RuntimeError( + "Failed to determine kernel start timestamp" + ) from e try: out, _ = subp.subp(['/bin/systemctl', @@ -116,12 +118,13 @@ def get_boot_telemetry(): user_start = kernel_start + (float(tsm) / 1000000) except subp.ProcessExecutionError as e: - raise RuntimeError("Failed to get UserspaceTimestampMonotonic: %s" - % e) + raise RuntimeError( + "Failed to get UserspaceTimestampMonotonic: %s" % e + ) from e except ValueError as e: - raise RuntimeError("Failed to parse " - "UserspaceTimestampMonotonic from systemd: %s" - % e) + raise RuntimeError( + "Failed to parse UserspaceTimestampMonotonic from systemd: %s" % e + ) from e try: out, _ = subp.subp(['/bin/systemctl', 'show', @@ -137,12 +140,14 @@ def get_boot_telemetry(): cloudinit_activation = kernel_start + (float(tsm) / 1000000) except subp.ProcessExecutionError as e: - raise RuntimeError("Failed to get InactiveExitTimestampMonotonic: %s" - % e) + raise RuntimeError( + "Failed to get InactiveExitTimestampMonotonic: %s" % e + ) from e except ValueError as e: - raise RuntimeError("Failed to parse " - "InactiveExitTimestampMonotonic from systemd: %s" - % e) + raise RuntimeError( + "Failed to parse InactiveExitTimestampMonotonic from systemd: %s" + % e + ) from e evt = events.ReportingEvent( BOOT_EVENT_TYPE, 'boot-telemetry', @@ -642,9 +647,10 @@ class WALinuxAgentShim: try: name = os.path.basename(hook_file).replace('.json', '') dhcp_options[name] = json.loads(util.load_file((hook_file))) - except ValueError: + except ValueError as e: raise ValueError( - '{_file} is not valid JSON data'.format(_file=hook_file)) + '{_file} is not valid JSON data'.format(_file=hook_file) + ) from e return dhcp_options @staticmethod diff --git a/cloudinit/sources/helpers/netlink.py b/cloudinit/sources/helpers/netlink.py index a74a3588..c2ad587b 100644 --- a/cloudinit/sources/helpers/netlink.py +++ b/cloudinit/sources/helpers/netlink.py @@ -74,7 +74,7 @@ def create_bound_netlink_socket(): netlink_socket.setblocking(0) except socket.error as e: msg = "Exception during netlink socket create: %s" % e - raise NetlinkCreateSocketError(msg) + raise NetlinkCreateSocketError(msg) from e LOG.debug("Created netlink socket") return netlink_socket diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 1050efb0..65e020c5 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -280,8 +280,9 @@ class BaseReader(metaclass=abc.ABCMeta): try: data = translator(data) except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to process path %s: %s" % (path, e) + ) from e if found: results[name] = data @@ -291,8 +292,9 @@ class BaseReader(metaclass=abc.ABCMeta): try: metadata['random_seed'] = base64.b64decode(random_seed) except (ValueError, TypeError) as e: - raise BrokenMetadata("Badly formatted metadata" - " random_seed entry: %s" % e) + raise BrokenMetadata( + "Badly formatted metadata random_seed entry: %s" % e + ) from e # load any files that were provided files = {} @@ -304,8 +306,9 @@ class BaseReader(metaclass=abc.ABCMeta): try: files[path] = self._read_content_path(item) except Exception as e: - raise BrokenMetadata("Failed to read provided " - "file %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to read provided file %s: %s" % (path, e) + ) from e results['files'] = files # The 'network_config' item in metadata is a content pointer @@ -317,8 +320,9 @@ class BaseReader(metaclass=abc.ABCMeta): content = self._read_content_path(net_item, decode=True) results['network_config'] = content except IOError as e: - raise BrokenMetadata("Failed to read network" - " configuration: %s" % (e)) + raise BrokenMetadata( + "Failed to read network configuration: %s" % (e) + ) from e # To openstack, user can specify meta ('nova boot --meta=key=value') # and those will appear under metadata['meta']. @@ -370,8 +374,9 @@ class ConfigDriveReader(BaseReader): try: return util.load_json(self._path_read(path)) except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to process path %s: %s" % (path, e) + ) from e def read_v1(self): """Reads a version 1 formatted location. @@ -395,16 +400,17 @@ class ConfigDriveReader(BaseReader): path = found[name] try: contents = self._path_read(path) - except IOError: - raise BrokenMetadata("Failed to read: %s" % path) + except IOError as e: + raise BrokenMetadata("Failed to read: %s" % path) from e try: # Disable not-callable pylint check; pylint isn't able to # determine that every member of FILES_V1 has a callable in # the appropriate position md[key] = translator(contents) # pylint: disable=E1102 except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to process path %s: %s" % (path, e) + ) from e else: md[key] = copy.deepcopy(default) diff --git a/cloudinit/subp.py b/cloudinit/subp.py index 804ef3ca..3e4efa42 100644 --- a/cloudinit/subp.py +++ b/cloudinit/subp.py @@ -262,7 +262,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True, raise ProcessExecutionError( cmd=args, reason=e, errno=e.errno, stdout="-" if decode else b"-", - stderr="-" if decode else b"-") + stderr="-" if decode else b"-" + ) from e finally: if devnull_fp: devnull_fp.close() diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index f3c0cf9c..caa88435 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -95,7 +95,7 @@ def read_file_or_url(url, **kwargs): code = e.errno if e.errno == ENOENT: code = NOT_FOUND - raise UrlError(cause=e, code=code, headers=None, url=url) + raise UrlError(cause=e, code=code, headers=None, url=url) from e return FileResponse(file_path, contents=contents) else: return readurl(url, **kwargs) @@ -575,8 +575,8 @@ def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret, timestamp=None): try: import oauthlib.oauth1 as oauth1 - except ImportError: - raise NotImplementedError('oauth support is not available') + except ImportError as e: + raise NotImplementedError('oauth support is not available') from e if timestamp: timestamp = str(timestamp) diff --git a/cloudinit/util.py b/cloudinit/util.py index dd263803..cf9e349f 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -359,7 +359,7 @@ def decomp_gzip(data, quiet=True, decode=True): if quiet: return data else: - raise DecompressionError(str(e)) + raise DecompressionError(str(e)) from e def extract_usergroup(ug_pair): @@ -1363,7 +1363,7 @@ def chownbyname(fname, user=None, group=None): if group: gid = grp.getgrnam(group).gr_gid except KeyError as e: - raise OSError("Unknown user or group: %s" % (e)) + raise OSError("Unknown user or group: %s" % (e)) from e chownbyid(fname, uid, gid) @@ -2387,8 +2387,8 @@ def human2bytes(size): try: num = float(num) - except ValueError: - raise ValueError("'%s' is not valid input." % size_in) + except ValueError as e: + raise ValueError("'%s' is not valid input." % size_in) from e if num < 0: raise ValueError("'%s': cannot be negative" % size_in) diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py index a136cf0d..eedbaae8 100644 --- a/tests/cloud_tests/platforms/azurecloud/instance.py +++ b/tests/cloud_tests/platforms/azurecloud/instance.py @@ -134,9 +134,10 @@ class AzureCloudInstance(Instance): self.vm_name, vm_params) LOG.debug('creating instance %s from image_id=%s', self.vm_name, self.image_id) - except CloudError: - raise RuntimeError('failed creating instance:\n{}'.format( - traceback.format_exc())) + except CloudError as e: + raise RuntimeError( + 'failed creating instance:\n{}'.format(traceback.format_exc()) + ) from e if wait: self.instance.wait() diff --git a/tests/cloud_tests/platforms/azurecloud/platform.py b/tests/cloud_tests/platforms/azurecloud/platform.py index cb62a74b..a664f612 100644 --- a/tests/cloud_tests/platforms/azurecloud/platform.py +++ b/tests/cloud_tests/platforms/azurecloud/platform.py @@ -59,9 +59,12 @@ class AzureCloudPlatform(Platform): self.vnet = self._create_vnet() self.subnet = self._create_subnet() self.nic = self._create_nic() - except CloudError: - raise RuntimeError('failed creating a resource:\n{}'.format( - traceback.format_exc())) + except CloudError as e: + raise RuntimeError( + 'failed creating a resource:\n{}'.format( + traceback.format_exc() + ) + ) from e def create_instance(self, properties, config, features, image_id, user_data=None): @@ -105,8 +108,10 @@ class AzureCloudPlatform(Platform): if image_id.find('__') > 0: image_id = image_id.split('__')[1] LOG.debug('image_id shortened to %s', image_id) - except KeyError: - raise RuntimeError('no images found for %s' % img_conf['release']) + except KeyError as e: + raise RuntimeError( + 'no images found for %s' % img_conf['release'] + ) from e return AzureCloudImage(self, img_conf, image_id) @@ -140,9 +145,11 @@ class AzureCloudPlatform(Platform): secret=azure_creds['clientSecret'], tenant=azure_creds['tenantId']) return credentials, subscription_id - except KeyError: - raise RuntimeError('Please configure Azure service principal' - ' credentials in %s' % cred_file) + except KeyError as e: + raise RuntimeError( + 'Please configure Azure service principal' + ' credentials in %s' % cred_file + ) from e def _create_resource_group(self): """Create resource group""" diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py index ab6037b1..d2e84047 100644 --- a/tests/cloud_tests/platforms/ec2/instance.py +++ b/tests/cloud_tests/platforms/ec2/instance.py @@ -49,11 +49,11 @@ class EC2Instance(Instance): # OutputBytes comes from platform._decode_console_output_as_bytes response = self.instance.console_output() return response['OutputBytes'] - except KeyError: + except KeyError as e: if 'Output' in response: msg = ("'OutputBytes' did not exist in console_output() but " "'Output' did: %s..." % response['Output'][0:128]) - raise util.PlatformError('console_log', msg) + raise util.PlatformError('console_log', msg) from e return ('No Console Output [%s]' % self.instance).encode() def destroy(self): diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py index 7a3d0fe0..b61a2ffb 100644 --- a/tests/cloud_tests/platforms/ec2/platform.py +++ b/tests/cloud_tests/platforms/ec2/platform.py @@ -35,12 +35,14 @@ class EC2Platform(Platform): self.ec2_resource = b3session.resource('ec2') self.ec2_region = b3session.region_name self.key_name = self._upload_public_key(config) - except botocore.exceptions.NoRegionError: + except botocore.exceptions.NoRegionError as e: raise RuntimeError( - 'Please configure default region in $HOME/.aws/config') - except botocore.exceptions.NoCredentialsError: + 'Please configure default region in $HOME/.aws/config' + ) from e + except botocore.exceptions.NoCredentialsError as e: raise RuntimeError( - 'Please configure ec2 credentials in $HOME/.aws/credentials') + 'Please configure ec2 credentials in $HOME/.aws/credentials' + ) from e self.vpc = self._create_vpc() self.internet_gateway = self._create_internet_gateway() @@ -125,8 +127,10 @@ class EC2Platform(Platform): try: image_ami = image['id'] - except KeyError: - raise RuntimeError('No images found for %s!' % img_conf['release']) + except KeyError as e: + raise RuntimeError( + 'No images found for %s!' % img_conf['release'] + ) from e LOG.debug('found image: %s', image_ami) image = EC2Image(self, img_conf, image_ami) @@ -195,7 +199,7 @@ class EC2Platform(Platform): CidrBlock=self.ipv4_cidr, AmazonProvidedIpv6CidrBlock=True) except botocore.exceptions.ClientError as e: - raise RuntimeError(e) + raise RuntimeError(e) from e vpc.wait_until_available() self._tag_resource(vpc) diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py index b27b9848..2b973a08 100644 --- a/tests/cloud_tests/platforms/lxd/instance.py +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -175,7 +175,8 @@ class LXDInstance(Instance): raise PlatformError( "console log", "Console log failed [%d]: stdout=%s stderr=%s" % ( - e.exit_code, e.stdout, e.stderr)) + e.exit_code, e.stdout, e.stderr) + ) from e def reboot(self, wait=True): """Reboot instance.""" diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py index 58f65e52..ac3b6563 100644 --- a/tests/cloud_tests/platforms/platforms.py +++ b/tests/cloud_tests/platforms/platforms.py @@ -74,8 +74,10 @@ class Platform(object): try: return tmirror.json_entries[0] - except IndexError: - raise RuntimeError('no images found with filter: %s' % img_filter) + except IndexError as e: + raise RuntimeError( + 'no images found with filter: %s' % img_filter + ) from e class FilterMirror(mirrors.BasicMirrorWriter): diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py index e8c371ca..bb9785d3 100644 --- a/tests/cloud_tests/testcases/__init__.py +++ b/tests/cloud_tests/testcases/__init__.py @@ -21,8 +21,10 @@ def discover_test(test_name): config.name_sanitize(test_name)) try: testmod = importlib.import_module(testmod_name) - except NameError: - raise ValueError('no test verifier found at: {}'.format(testmod_name)) + except NameError as e: + raise ValueError( + 'no test verifier found at: {}'.format(testmod_name) + ) from e found = [mod for name, mod in inspect.getmembers(testmod) if (inspect.isclass(mod) diff --git a/tools/mock-meta.py b/tools/mock-meta.py index a58e0260..9dd067b9 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -258,12 +258,14 @@ class MetaDataHandler(object): try: key_id = int(mybe_key) key_name = key_ids[key_id] - except ValueError: - raise WebException(hclient.BAD_REQUEST, - "%s: not an integer" % mybe_key) - except IndexError: - raise WebException(hclient.NOT_FOUND, - "Unknown key id %r" % mybe_key) + except ValueError as e: + raise WebException( + hclient.BAD_REQUEST, "%s: not an integer" % mybe_key + ) from e + except IndexError as e: + raise WebException( + hclient.NOT_FOUND, "Unknown key id %r" % mybe_key + ) from e # Extract the possible sub-params result = traverse(nparams[1:], { "openssh-key": "\n".join(avail_keys[key_name]), diff --git a/tox.ini b/tox.ini index f619dbf5..a92c63e0 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ setenv = basepython = python3 deps = # requirements - pylint==2.5.3 + pylint==2.6.0 # test-requirements because unit tests are now present in cloudinit tree -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt -- cgit v1.2.3