From 71af48df3514ca831c90b77dc71ba0a121dec401 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 10 Mar 2020 08:22:22 -0600 Subject: instance-data: add cloud-init merged_cfg and sys_info keys to json (#214) Cloud-config userdata provided as jinja templates are now distro, platform and merged cloud config aware. The cloud-init query command will also surface this config data. Now users can selectively render portions of cloud-config based on: * distro name, version, release * python version * merged cloud config values * machine platform * kernel To support template handling of this config, add new top-level keys to /run/cloud-init/instance-data.json. The new 'merged_cfg' key represents merged cloud config from /etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/*. The new 'sys_info' key which captures distro and platform info from cloudinit.util.system_info. Cloud config userdata templates can render conditional content based on these additional environmental checks such as the following simple example: ``` ## template: jinja #cloud-config runcmd: {% if distro == 'opensuse' %} - sh /custom-setup-sles {% elif distro == 'centos' %} - sh /custom-setup-centos {% elif distro == 'debian' %} - sh /custom-setup-debian {% endif %} ``` To see all values: sudo cloud-init query --all Any keys added to the standardized v1 keys are guaranteed to not change or drop on future released of cloud-init. 'v1' keys will be retained for backward-compatibility even if a new standardized 'v2' set of keys are introduced The following standardized v1 keys are added: * distro, distro_release, distro_version, kernel_version, machine, python_version, system_platform, variant LP: #1865969 --- tests/cloud_tests/testcases/base.py | 55 +++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) (limited to 'tests/cloud_tests/testcases') diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index fd12d87b..5976e234 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -172,9 +172,7 @@ class CloudTestCase(unittest2.TestCase): 'Skipping instance-data.json test.' ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) - self.assertItemsEqual( - [], - instance_data['base64_encoded_keys']) + self.assertItemsEqual(['ci_cfg'], instance_data['sensitive_keys']) ds = instance_data.get('ds', {}) v1_data = instance_data.get('v1', {}) metadata = ds.get('meta-data', {}) @@ -201,6 +199,23 @@ class CloudTestCase(unittest2.TestCase): self.assertIn('i-', v1_data['instance_id']) self.assertIn('ip-', v1_data['local_hostname']) self.assertIsNotNone(v1_data['region'], 'expected ec2 region') + self.assertIsNotNone( + re.match(r'\d\.\d+\.\d+-\d+-aws', v1_data['kernel_release'])) + self.assertEqual( + 'redacted for non-root user', instance_data['merged_cfg']) + self.assertEqual(self.os_cfg['os'], v1_data['variant']) + self.assertEqual(self.os_cfg['os'], v1_data['distro']) + self.assertEqual( + self.os_cfg['os'], instance_data["sys_info"]['dist'][0], + "Unexpected sys_info dist value") + self.assertEqual(self.os_name, v1_data['distro_release']) + self.assertEqual( + str(self.os_cfg['version']), v1_data['distro_version']) + self.assertEqual('x86_64', v1_data['machine']) + self.assertIsNotNone( + re.match(r'3.\d\.\d', v1_data['python_version']), + "unexpected python version: {ver}".format( + ver=v1_data["python_version"])) def test_instance_data_json_lxd(self): """Validate instance-data.json content by lxd platform. @@ -237,6 +252,23 @@ class CloudTestCase(unittest2.TestCase): self.assertIsNone( v1_data['region'], 'found unexpected lxd region %s' % v1_data['region']) + self.assertIsNotNone( + re.match(r'\d\.\d+\.\d+-\d+', v1_data['kernel_release'])) + self.assertEqual( + 'redacted for non-root user', instance_data['merged_cfg']) + self.assertEqual(self.os_cfg['os'], v1_data['variant']) + self.assertEqual(self.os_cfg['os'], v1_data['distro']) + self.assertEqual( + self.os_cfg['os'], instance_data["sys_info"]['dist'][0], + "Unexpected sys_info dist value") + self.assertEqual(self.os_name, v1_data['distro_release']) + self.assertEqual( + str(self.os_cfg['version']), v1_data['distro_version']) + self.assertEqual('x86_64', v1_data['machine']) + self.assertIsNotNone( + re.match(r'3.\d\.\d', v1_data['python_version']), + "unexpected python version: {ver}".format( + ver=v1_data["python_version"])) def test_instance_data_json_kvm(self): """Validate instance-data.json content by nocloud-kvm platform. @@ -278,6 +310,23 @@ class CloudTestCase(unittest2.TestCase): self.assertIsNone( v1_data['region'], 'found unexpected lxd region %s' % v1_data['region']) + self.assertIsNotNone( + re.match(r'\d\.\d+\.\d+-\d+', v1_data['kernel_release'])) + self.assertEqual( + 'redacted for non-root user', instance_data['merged_cfg']) + self.assertEqual(self.os_cfg['os'], v1_data['variant']) + self.assertEqual(self.os_cfg['os'], v1_data['distro']) + self.assertEqual( + self.os_cfg['os'], instance_data["sys_info"]['dist'][0], + "Unexpected sys_info dist value") + self.assertEqual(self.os_name, v1_data['distro_release']) + self.assertEqual( + str(self.os_cfg['version']), v1_data['distro_version']) + self.assertEqual('x86_64', v1_data['machine']) + self.assertIsNotNone( + re.match(r'3.\d\.\d', v1_data['python_version']), + "unexpected python version: {ver}".format( + ver=v1_data["python_version"])) class PasswordListTest(CloudTestCase): -- cgit v1.2.3 From 024bf27b5a3880dac916431296cb871707923562 Mon Sep 17 00:00:00 2001 From: Paride Legovini Date: Wed, 18 Mar 2020 18:47:33 +0100 Subject: ec2 json validation: fix the reference to the 'merged_cfg' key (#256) The 'merged_cfg' key introduced in 71af48d was incorrectly referenced to as 'ci_cfg' in the json validation test for ec2. --- tests/cloud_tests/testcases/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/cloud_tests/testcases') diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 5976e234..b0dfc144 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -172,7 +172,7 @@ class CloudTestCase(unittest2.TestCase): 'Skipping instance-data.json test.' ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) - self.assertItemsEqual(['ci_cfg'], instance_data['sensitive_keys']) + self.assertItemsEqual(['merged_cfg'], instance_data['sensitive_keys']) ds = instance_data.get('ds', {}) v1_data = instance_data.get('v1', {}) metadata = ds.get('meta-data', {}) -- cgit v1.2.3 From 38a7e6e9756fdab31264c0d6e93d20432ed111ac Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 24 Apr 2020 09:26:51 -0400 Subject: cloudinit: drop dependencies on unittest2 and contextlib2 (#322) These libraries provide backports of Python 3's stdlib components to Python 2. As we only support Python 3, we can simply use the stdlib now. This pull request does the following: * removes some unneeded compatibility code for the old spelling of `assertRaisesRegex` * replaces invocations of the Python 2-only `assertItemsEqual` with its new name, `assertCountEqual` * replaces all usage of `unittest2` with `unittest` * replaces all usage of `contextlib2` with `contextlib` * drops `unittest2` and `contextlib2` from requirements files and tox.ini It also rewrites some `test_azure` helpers to use bare asserts. We were seeing a strange error in xenial builds of this branch which appear to be stemming from the AssertionError that pytest produces being _different_ from the standard AssertionError. This means that the modified helpers weren't behaving correctly, because they weren't catching AssertionErrors as one would expect. (I believe this is related, in some way, to https://github.com/pytest-dev/pytest/issues/645, but the only version of pytest where we're affected is so far in the past that it's not worth pursuing it any further as we have a workaround.) --- cloudinit/config/tests/test_users_groups.py | 10 ++++---- cloudinit/net/tests/test_dhcp.py | 10 ++++---- cloudinit/net/tests/test_init.py | 2 +- cloudinit/sources/tests/test_init.py | 10 ++++---- cloudinit/tests/helpers.py | 28 ++++++---------------- integration-requirements.txt | 1 - packages/pkg-deps.json | 3 --- test-requirements.txt | 2 -- tests/cloud_tests/testcases/__init__.py | 8 +++---- tests/cloud_tests/testcases/base.py | 12 +++++----- tests/cloud_tests/testcases/modules/ntp_chrony.py | 4 ++-- tests/cloud_tests/verify.py | 4 ++-- tests/unittests/test_builtin_handlers.py | 6 ++--- tests/unittests/test_datasource/test_azure.py | 8 +++---- .../unittests/test_datasource/test_azure_helper.py | 6 ++--- tests/unittests/test_datasource/test_smartos.py | 12 +++++----- .../test_handler/test_handler_ca_certs.py | 6 +---- tests/unittests/test_handler/test_handler_chef.py | 8 +++---- .../test_handler/test_handler_growpart.py | 6 +---- tests/unittests/test_handler/test_schema.py | 4 ++-- tox.ini | 2 -- 21 files changed, 61 insertions(+), 91 deletions(-) (limited to 'tests/cloud_tests/testcases') diff --git a/cloudinit/config/tests/test_users_groups.py b/cloudinit/config/tests/test_users_groups.py index f620b597..df89ddb3 100644 --- a/cloudinit/config/tests/test_users_groups.py +++ b/cloudinit/config/tests/test_users_groups.py @@ -39,7 +39,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -65,7 +65,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='freebsd', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_fbsd_user.call_args_list, [mock.call('freebsd', groups='wheel', lock_passwd=True, shell='/bin/tcsh'), @@ -86,7 +86,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -107,7 +107,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), @@ -146,7 +146,7 @@ class TestHandleUsersGroups(CiTestCase): cloud = self.tmp_cloud( distro='ubuntu', sys_cfg=sys_cfg, metadata=metadata) cc_users_groups.handle('modulename', cfg, cloud, None, None) - self.assertItemsEqual( + self.assertCountEqual( m_user.call_args_list, [mock.call('ubuntu', groups='lxd,sudo', lock_passwd=True, shell='/bin/bash'), diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index c3fa1e04..bc7bef45 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -62,7 +62,7 @@ class TestParseDHCPLeasesFile(CiTestCase): {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) class TestDHCPRFC3442(CiTestCase): @@ -88,7 +88,7 @@ class TestDHCPRFC3442(CiTestCase): 'renew': '4 2017/07/27 18:02:30', 'expire': '5 2017/07/28 07:08:15'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) def test_parse_lease_finds_classless_static_routes(self): """ @@ -114,7 +114,7 @@ class TestDHCPRFC3442(CiTestCase): 'renew': '4 2017/07/27 18:02:30', 'expire': '5 2017/07/28 07:08:15'}] write_file(lease_file, content) - self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file)) @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network') @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') @@ -324,7 +324,7 @@ class TestDHCPDiscoveryClean(CiTestCase): """) write_file(self.tmp_path('dhcp.leases', tmpdir), lease_content) - self.assertItemsEqual( + self.assertCountEqual( [{'interface': 'eth9', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], dhcp_discovery(dhclient_script, 'eth9', tmpdir)) @@ -389,7 +389,7 @@ class TestDHCPDiscoveryClean(CiTestCase): write_file(pid_file, "%d\n" % my_pid) m_getppid.return_value = 1 # Indicate that dhclient has daemonized - self.assertItemsEqual( + self.assertCountEqual( [{'interface': 'eth9', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], dhcp_discovery(dhclient_script, 'eth9', tmpdir)) diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index 32e70b4c..835ed807 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -397,7 +397,7 @@ class TestGetDeviceList(CiTestCase): """get_devicelist returns a directory listing for SYS_CLASS_NET.""" write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), 'up') write_file(os.path.join(self.sysdir, 'eth1', 'operstate'), 'up') - self.assertItemsEqual(['eth0', 'eth1'], net.get_devicelist()) + self.assertCountEqual(['eth0', 'eth1'], net.get_devicelist()) class TestGetInterfaceMAC(CiTestCase): diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py index 5b3648e9..5b6f1b3f 100644 --- a/cloudinit/sources/tests/test_init.py +++ b/cloudinit/sources/tests/test_init.py @@ -350,7 +350,7 @@ class TestDataSource(CiTestCase): 'region': 'myregion', 'some': {'security-credentials': { 'cred1': 'sekret', 'cred2': 'othersekret'}}}) - self.assertItemsEqual( + self.assertCountEqual( ('merged_cfg', 'security-credentials',), datasource.sensitive_metadata_keys) sys_info = { @@ -401,7 +401,7 @@ class TestDataSource(CiTestCase): 'region': 'myregion', 'some': {'security-credentials': REDACT_SENSITIVE_VALUE}}} } - self.assertItemsEqual(expected, redacted) + self.assertCountEqual(expected, redacted) file_stat = os.stat(json_file) self.assertEqual(0o644, stat.S_IMODE(file_stat.st_mode)) @@ -426,7 +426,7 @@ class TestDataSource(CiTestCase): "x86_64"], "variant": "ubuntu", "dist": ["ubuntu", "20.04", "focal"]} - self.assertItemsEqual( + self.assertCountEqual( ('merged_cfg', 'security-credentials',), datasource.sensitive_metadata_keys) with mock.patch("cloudinit.util.system_info", return_value=sys_info): @@ -476,7 +476,7 @@ class TestDataSource(CiTestCase): 'security-credentials': {'cred1': 'sekret', 'cred2': 'othersekret'}}}} } - self.assertItemsEqual(expected, util.load_json(content)) + self.assertCountEqual(expected, util.load_json(content)) file_stat = os.stat(sensitive_json_file) self.assertEqual(0o600, stat.S_IMODE(file_stat.st_mode)) self.assertEqual(expected, util.load_json(content)) @@ -542,7 +542,7 @@ class TestDataSource(CiTestCase): json_file = self.tmp_path(INSTANCE_JSON_FILE, tmp) content = util.load_file(json_file) instance_json = util.load_json(content) - self.assertItemsEqual( + self.assertCountEqual( ['ds/meta_data/key2/key2.1'], instance_json['base64_encoded_keys']) self.assertEqual( diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index f4db5827..477e14c2 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -13,15 +13,10 @@ import string import sys import tempfile import time +import unittest +from contextlib import ExitStack, contextmanager from unittest import mock - -import unittest2 -from unittest2.util import strclass - -try: - from contextlib import ExitStack, contextmanager -except ImportError: - from contextlib2 import ExitStack, contextmanager +from unittest.util import strclass from cloudinit.config.schema import ( SchemaValidationError, validate_cloudconfig_schema) @@ -35,8 +30,8 @@ from cloudinit import util _real_subp = util.subp # Used for skipping tests -SkipTest = unittest2.SkipTest -skipIf = unittest2.skipIf +SkipTest = unittest.SkipTest +skipIf = unittest.skipIf # Makes the old path start @@ -73,7 +68,7 @@ def retarget_many_wrapper(new_base, am, old_func): return wrapper -class TestCase(unittest2.TestCase): +class TestCase(unittest.TestCase): def reset_global_state(self): """Reset any global state to its original settings. @@ -372,7 +367,7 @@ class HttprettyTestCase(CiTestCase): super(HttprettyTestCase, self).tearDown() -class SchemaTestCaseMixin(unittest2.TestCase): +class SchemaTestCaseMixin(unittest.TestCase): def assertSchemaValid(self, cfg, msg="Valid Schema failed validation."): """Assert the config is valid per self.schema. @@ -504,13 +499,4 @@ if not hasattr(mock.Mock, 'assert_not_called'): raise AssertionError(msg) mock.Mock.assert_not_called = __mock_assert_not_called - -# older unittest2.TestCase (centos6) have only the now-deprecated -# assertRaisesRegexp. Simple assignment makes pylint complain, about -# users of assertRaisesRegex so we use getattr to trick it. -# https://github.com/PyCQA/pylint/issues/1946 -if not hasattr(unittest2.TestCase, 'assertRaisesRegex'): - unittest2.TestCase.assertRaisesRegex = ( - getattr(unittest2.TestCase, 'assertRaisesRegexp')) - # vi: ts=4 expandtab diff --git a/integration-requirements.txt b/integration-requirements.txt index 897d6110..44e45c1b 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -5,7 +5,6 @@ # the packages/pkg-deps.json file as well. # -unittest2 # ec2 backend boto3==1.5.9 diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json index d09ec33f..f02e8348 100644 --- a/packages/pkg-deps.json +++ b/packages/pkg-deps.json @@ -10,9 +10,6 @@ "2" : "python-yaml", "3" : "python3-yaml" }, - "contextlib2" : { - "2" : "python-contextlib2" - }, "pyserial" : { "2" : "python-serial", "3" : "python3-serial" diff --git a/test-requirements.txt b/test-requirements.txt index 057115b6..0a6a04d4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,9 +1,7 @@ # Needed generally in tests httpretty>=0.7.1 pytest -unittest2 pytest-cov # Only really needed on older versions of python -contextlib2 setuptools diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py index 6bb39f77..e8c371ca 100644 --- a/tests/cloud_tests/testcases/__init__.py +++ b/tests/cloud_tests/testcases/__init__.py @@ -4,7 +4,7 @@ import importlib import inspect -import unittest2 +import unittest from cloudinit.util import read_conf @@ -48,7 +48,7 @@ def get_test_class(test_name, test_data, test_conf): def __str__(self): return "%s (%s)" % (self._testMethodName, - unittest2.util.strclass(self._realclass)) + unittest.util.strclass(self._realclass)) @classmethod def setUpClass(cls): @@ -62,9 +62,9 @@ def get_suite(test_name, data, conf): @return_value: a test suite """ - suite = unittest2.TestSuite() + suite = unittest.TestSuite() suite.addTest( - unittest2.defaultTestLoader.loadTestsFromTestCase( + unittest.defaultTestLoader.loadTestsFromTestCase( get_test_class(test_name, data, conf))) return suite diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index b0dfc144..7b67f54e 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -5,15 +5,15 @@ import crypt import json import re -import unittest2 +import unittest from cloudinit import util as c_util -SkipTest = unittest2.SkipTest +SkipTest = unittest.SkipTest -class CloudTestCase(unittest2.TestCase): +class CloudTestCase(unittest.TestCase): """Base test class for verifiers.""" # data gets populated in get_suite.setUpClass @@ -172,7 +172,7 @@ class CloudTestCase(unittest2.TestCase): 'Skipping instance-data.json test.' ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) - self.assertItemsEqual(['merged_cfg'], instance_data['sensitive_keys']) + self.assertCountEqual(['merged_cfg'], instance_data['sensitive_keys']) ds = instance_data.get('ds', {}) v1_data = instance_data.get('v1', {}) metadata = ds.get('meta-data', {}) @@ -237,7 +237,7 @@ class CloudTestCase(unittest2.TestCase): ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) v1_data = instance_data.get('v1', {}) - self.assertItemsEqual([], sorted(instance_data['base64_encoded_keys'])) + self.assertCountEqual([], sorted(instance_data['base64_encoded_keys'])) self.assertEqual('unknown', v1_data['cloud_name']) self.assertEqual('lxd', v1_data['platform']) self.assertEqual( @@ -291,7 +291,7 @@ class CloudTestCase(unittest2.TestCase): ' OS: %s not bionic or newer' % self.os_name) instance_data = json.loads(out) v1_data = instance_data.get('v1', {}) - self.assertItemsEqual([], instance_data['base64_encoded_keys']) + self.assertCountEqual([], instance_data['base64_encoded_keys']) self.assertEqual('unknown', v1_data['cloud_name']) self.assertEqual('nocloud', v1_data['platform']) subplatform = v1_data['subplatform'] diff --git a/tests/cloud_tests/testcases/modules/ntp_chrony.py b/tests/cloud_tests/testcases/modules/ntp_chrony.py index 0f4c3d08..7d341773 100644 --- a/tests/cloud_tests/testcases/modules/ntp_chrony.py +++ b/tests/cloud_tests/testcases/modules/ntp_chrony.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. """cloud-init Integration Test Verify Script.""" -import unittest2 +import unittest from tests.cloud_tests.testcases import base @@ -13,7 +13,7 @@ class TestNtpChrony(base.CloudTestCase): """Skip this suite of tests on lxd and artful or older.""" if self.platform == 'lxd': if self.is_distro('ubuntu') and self.os_version_cmp('artful') <= 0: - raise unittest2.SkipTest( + raise unittest.SkipTest( 'No support for chrony on containers <= artful.' ' LP: #1589780') return super(TestNtpChrony, self).setUp() diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py index 7018f4d5..0295af40 100644 --- a/tests/cloud_tests/verify.py +++ b/tests/cloud_tests/verify.py @@ -3,7 +3,7 @@ """Verify test results.""" import os -import unittest2 +import unittest from tests.cloud_tests import (config, LOG, util, testcases) @@ -18,7 +18,7 @@ def verify_data(data_dir, platform, os_name, tests): @return_value: {: {passed: True/False, failures: []}} """ base_dir = os.sep.join((data_dir, platform, os_name)) - runner = unittest2.TextTestRunner(verbosity=util.current_verbosity()) + runner = unittest.TextTestRunner(verbosity=util.current_verbosity()) res = {} for test_name in tests: LOG.debug('verifying test data for %s', test_name) diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index b92ffc79..9045e743 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -109,7 +109,7 @@ class TestJinjaTemplatePartHandler(CiTestCase): cloudconfig_handler = CloudConfigPartHandler(self.paths) h = JinjaTemplatePartHandler( self.paths, sub_handlers=[script_handler, cloudconfig_handler]) - self.assertItemsEqual( + self.assertCountEqual( ['text/cloud-config', 'text/cloud-config-jsonp', 'text/x-shellscript'], h.sub_handlers) @@ -120,7 +120,7 @@ class TestJinjaTemplatePartHandler(CiTestCase): cloudconfig_handler = CloudConfigPartHandler(self.paths) h = JinjaTemplatePartHandler( self.paths, sub_handlers=[script_handler, cloudconfig_handler]) - self.assertItemsEqual( + self.assertCountEqual( ['text/cloud-config', 'text/cloud-config-jsonp', 'text/x-shellscript'], h.sub_handlers) @@ -302,7 +302,7 @@ class TestConvertJinjaInstanceData(CiTestCase): expected_data.update({'v1key1': 'v1.1', 'v2key1': 'v2.1'}) converted_data = convert_jinja_instance_data(data=data) - self.assertItemsEqual( + self.assertCountEqual( ['ds', 'v1', 'v2', 'v1key1', 'v2key1'], converted_data.keys()) self.assertEqual( expected_data, diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 2c6aa44d..2f8561cb 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -528,14 +528,14 @@ scbus-1 on xpt0 bus 0 def tags_exists(x, y): for tag in x.keys(): - self.assertIn(tag, y) + assert tag in y for tag in y.keys(): - self.assertIn(tag, x) + assert tag in x def tags_equal(x, y): for x_val in x.values(): y_val = y.get(x_val.tag) - self.assertEqual(x_val.text, y_val.text) + assert x_val.text == y_val.text old_cnt = create_tag_index(oxml) new_cnt = create_tag_index(nxml) @@ -649,7 +649,7 @@ scbus-1 on xpt0 bus 0 crawled_metadata = dsrc.crawl_metadata() - self.assertItemsEqual( + self.assertCountEqual( crawled_metadata.keys(), ['cfg', 'files', 'metadata', 'userdata_raw']) self.assertEqual(crawled_metadata['cfg'], expected_cfg) diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 007df09f..6344b0bb 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import os -import unittest2 +import unittest from textwrap import dedent from cloudinit.sources.helpers import azure as azure_helper @@ -332,7 +332,7 @@ class TestOpenSSLManagerActions(CiTestCase): path = 'tests/data/azure' return os.path.join(path, name) - @unittest2.skip("todo move to cloud_test") + @unittest.skip("todo move to cloud_test") def test_pubkey_extract(self): cert = load_file(self._data_file('pubkey_extract_cert')) good_key = load_file(self._data_file('pubkey_extract_ssh_key')) @@ -344,7 +344,7 @@ class TestOpenSSLManagerActions(CiTestCase): fingerprint = sslmgr._get_fingerprint_from_cert(cert) self.assertEqual(good_fingerprint, fingerprint) - @unittest2.skip("todo move to cloud_test") + @unittest.skip("todo move to cloud_test") @mock.patch.object(azure_helper.OpenSSLManager, '_decrypt_certs_from_xml') def test_parse_certificates(self, mock_decrypt_certs): """Azure control plane puts private keys as well as certificates diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 62084de5..0f0b42bb 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -22,7 +22,7 @@ import os.path import re import signal import stat -import unittest2 +import unittest import uuid from cloudinit import serial @@ -1095,11 +1095,11 @@ class TestNetworkConversion(CiTestCase): self.assertEqual(expected, found) -@unittest2.skipUnless(get_smartos_environ() == SMARTOS_ENV_KVM, - "Only supported on KVM and bhyve guests under SmartOS") -@unittest2.skipUnless(os.access(SERIAL_DEVICE, os.W_OK), - "Requires write access to " + SERIAL_DEVICE) -@unittest2.skipUnless(HAS_PYSERIAL is True, "pyserial not available") +@unittest.skipUnless(get_smartos_environ() == SMARTOS_ENV_KVM, + "Only supported on KVM and bhyve guests under SmartOS") +@unittest.skipUnless(os.access(SERIAL_DEVICE, os.W_OK), + "Requires write access to " + SERIAL_DEVICE) +@unittest.skipUnless(HAS_PYSERIAL is True, "pyserial not available") class TestSerialConcurrency(CiTestCase): """ This class tests locking on an actual serial port, and as such can only diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index 5b4105dd..286ef771 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -11,13 +11,9 @@ import logging import shutil import tempfile import unittest +from contextlib import ExitStack from unittest import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - class TestNoConfig(unittest.TestCase): def setUp(self): diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 2dab3a54..513c18b5 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -65,18 +65,18 @@ class TestInstallChefOmnibus(HttprettyTestCase): cc_chef.install_chef_from_omnibus() expected_kwargs = {'retries': cc_chef.OMNIBUS_URL_RETRIES, 'url': cc_chef.OMNIBUS_URL} - self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[0][1]) + self.assertCountEqual(expected_kwargs, m_rdurl.call_args_list[0][1]) cc_chef.install_chef_from_omnibus(retries=10) expected_kwargs = {'retries': 10, 'url': cc_chef.OMNIBUS_URL} - self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[1][1]) + self.assertCountEqual(expected_kwargs, m_rdurl.call_args_list[1][1]) expected_subp_kwargs = { 'args': ['-v', '2.0'], 'basename': 'chef-omnibus-install', 'blob': m_rdurl.return_value.contents, 'capture': False } - self.assertItemsEqual( + self.assertCountEqual( expected_subp_kwargs, m_subp_blob.call_args_list[0][1]) @@ -97,7 +97,7 @@ class TestInstallChefOmnibus(HttprettyTestCase): 'blob': response, 'capture': False } - self.assertItemsEqual(expected_kwargs, called_kwargs) + self.assertCountEqual(expected_kwargs, called_kwargs) class TestChef(FilesystemMockingTestCase): diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index 43b53745..501bcca5 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -11,13 +11,9 @@ import logging import os import re import unittest +from contextlib import ExitStack from unittest import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - # growpart: # mode: auto # off, on, auto, 'growpart' # devices: ['root'] diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index abde02d3..98628120 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -20,7 +20,7 @@ class GetSchemaTest(CiTestCase): def test_get_schema_coalesces_known_schema(self): """Every cloudconfig module with schema is listed in allOf keyword.""" schema = get_schema() - self.assertItemsEqual( + self.assertCountEqual( [ 'cc_bootcmd', 'cc_ntp', @@ -38,7 +38,7 @@ class GetSchemaTest(CiTestCase): schema['$schema']) # FULL_SCHEMA is updated by the get_schema call from cloudinit.config.schema import FULL_SCHEMA - self.assertItemsEqual(['id', '$schema', 'allOf'], FULL_SCHEMA.keys()) + self.assertCountEqual(['id', '$schema', 'allOf'], FULL_SCHEMA.keys()) def test_get_schema_returns_global_when_set(self): """When FULL_SCHEMA global is already set, get_schema returns it.""" diff --git a/tox.ini b/tox.ini index 416ddcb8..906519c4 100644 --- a/tox.ini +++ b/tox.ini @@ -75,8 +75,6 @@ deps = # test-requirements httpretty==0.9.6 mock==1.3.0 - unittest2==1.1.0 - contextlib2==0.5.1 [testenv:xenial] # When updating this commands definition, also update the definition in -- cgit v1.2.3 From 2e32c40a607250bc9e713c0daf360dc6617f4420 Mon Sep 17 00:00:00 2001 From: lucasmoura Date: Wed, 13 May 2020 17:45:01 -0300 Subject: Add schema to apt configure config (#357) Create a schema object for the `apt_configure` module and validate this schema in the `handle` function of the module. There are some considerations regarding this PR: * The `primary` and `security` keys have the exact same properties. I tried to eliminate this redundancy by moving their properties to a common place and then just referencing it for both security and primary. Similar to what is documented here: https://json-schema.org/understanding-json-schema/structuring.html under the `Reuse` paragraph. However, this approach does not work, because the `#` pointer goes to the beginning of the file, which is a python module instead of a json file, not allowing the pointer to find the correct definition. What I did was to create a separate dict for the mirror config and reuse it for primary and security, but maybe there are better approaches to do that. * There was no documentation for the config `debconf_selections`. I tried to infer what it supposed to do by looking at the code and the `debconf-set-selections` manpage, but my description may not be accurate or complete. * Add a _parse_description function to schema.py to render multi-line preformatted content instead of squashing all whitespace LP: #1858884 --- cloudinit/config/cc_apt_configure.py | 538 +++++++++++++-------- cloudinit/config/cc_ntp.py | 42 +- cloudinit/config/cc_write_files.py | 22 +- cloudinit/config/schema.py | 24 +- doc/examples/cloud-config-apt.txt | 2 +- doc/examples/cloud-config-chef-oneiric.txt | 63 +-- doc/examples/cloud-config.txt | 9 +- .../examples/install_run_chef_recipes.yaml | 73 +-- tests/unittests/test_handler/test_schema.py | 36 ++ 9 files changed, 507 insertions(+), 302 deletions(-) (limited to 'tests/cloud_tests/testcases') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ff58c062..9a33451d 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -6,228 +6,371 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Apt Configure -------------- -**Summary:** configure apt - -This module handles both configuration of apt options and adding source lists. -There are configuration options such as ``apt_get_wrapper`` and -``apt_get_command`` that control how cloud-init invokes apt-get. -These configuration options are handled on a per-distro basis, so consult -documentation for cloud-init's distro support for instructions on using -these config options. - -.. note:: - To ensure that apt configuration is valid yaml, any strings containing - special characters, especially ``:`` should be quoted. - -.. note:: - For more information about apt configuration, see the - ``Additional apt configuration`` example. - -**Preserve sources.list:** - -By default, cloud-init will generate a new sources list in -``/etc/apt/sources.list.d`` based on any changes specified in cloud config. -To disable this behavior and preserve the sources list from the pristine image, -set ``preserve_sources_list`` to ``true``. - -.. note:: - The ``preserve_sources_list`` option overrides all other config keys that - would alter ``sources.list`` or ``sources.list.d``, **except** for - additional sources to be added to ``sources.list.d``. - -**Disable source suites:** - -Entries in the sources list can be disabled using ``disable_suites``, which -takes a list of suites to be disabled. If the string ``$RELEASE`` is present in -a suite in the ``disable_suites`` list, it will be replaced with the release -name. If a suite specified in ``disable_suites`` is not present in -``sources.list`` it will be ignored. For convenience, several aliases are -provided for ``disable_suites``: - - - ``updates`` => ``$RELEASE-updates`` - - ``backports`` => ``$RELEASE-backports`` - - ``security`` => ``$RELEASE-security`` - - ``proposed`` => ``$RELEASE-proposed`` - - ``release`` => ``$RELEASE`` - -.. note:: - When a suite is disabled using ``disable_suites``, its entry in - ``sources.list`` is not deleted; it is just commented out. - -**Configure primary and security mirrors:** - -The primary and security archive mirrors can be specified using the ``primary`` -and ``security`` keys, respectively. Both the ``primary`` and ``security`` keys -take a list of configs, allowing mirrors to be specified on a per-architecture -basis. Each config is a dictionary which must have an entry for ``arches``, -specifying which architectures that config entry is for. The keyword -``default`` applies to any architecture not explicitly listed. The mirror url -can be specified with the ``uri`` key, or a list of mirrors to check can be -provided in order, with the first mirror that can be resolved being selected. -This allows the same configuration to be used in different environment, with -different hosts used for a local apt mirror. If no mirror is provided by -``uri`` or ``search``, ``search_dns`` may be used to search for dns names in -the format ``-mirror`` in each of the following: - - - fqdn of this host per cloud metadata - - localdomain - - domains listed in ``/etc/resolv.conf`` - -If there is a dns entry for ``-mirror``, then it is assumed that there -is a distro mirror at ``http://-mirror./``. If the -``primary`` key is defined, but not the ``security`` key, then then -configuration for ``primary`` is also used for ``security``. If ``search_dns`` -is used for the ``security`` key, the search pattern will be. -``-security-mirror``. - -If no mirrors are specified, or all lookups fail, then default mirrors defined -in the datasource are used. If none are present in the datasource either the -following defaults are used: - - - primary: ``http://archive.ubuntu.com/ubuntu`` - - security: ``http://security.ubuntu.com/ubuntu`` - -**Specify sources.list template:** - -A custom template for rendering ``sources.list`` can be specefied with -``sources_list``. If no ``sources_list`` template is given, cloud-init will -use sane default. Within this template, the following strings will be replaced -with the appropriate values: - - - ``$MIRROR`` - - ``$RELEASE`` - - ``$PRIMARY`` - - ``$SECURITY`` - -**Pass configuration to apt:** - -Apt configuration can be specified using ``conf``. Configuration is specified -as a string. For multiline apt configuration, make sure to follow yaml syntax. - -**Configure apt proxy:** - -Proxy configuration for apt can be specified using ``conf``, but proxy config -keys also exist for convenience. The proxy config keys, ``http_proxy``, -``ftp_proxy``, and ``https_proxy`` may be used to specify a proxy for http, ftp -and https protocols respectively. The ``proxy`` key also exists as an alias for -``http_proxy``. Proxy url is specified in the format -``://[[user][:pass]@]host[:port]/``. - -**Add apt repos by regex:** +"""Apt Configure: Configure apt for the user.""" -All source entries in ``apt-sources`` that match regex in -``add_apt_repo_match`` will be added to the system using -``add-apt-repository``. If ``add_apt_repo_match`` is not specified, it defaults -to ``^[\\w-]+:\\w`` - -**Add source list entries:** - -Source list entries can be specified as a dictionary under the ``sources`` -config key, with key in the dict representing a different source file. The key -of each source entry will be used as an id that can be referenced in -other config entries, as well as the filename for the source's configuration -under ``/etc/apt/sources.list.d``. If the name does not end with ``.list``, -it will be appended. If there is no configuration for a key in ``sources``, no -file will be written, but the key may still be referred to as an id in other -``sources`` entries. - -Each entry under ``sources`` is a dictionary which may contain any of the -following optional keys: - - - ``source``: a sources.list entry (some variable replacements apply) - - ``keyid``: a key to import via shortid or fingerprint - - ``key``: a raw PGP key - - ``keyserver``: alternate keyserver to pull ``keyid`` key from - -The ``source`` key supports variable replacements for the following strings: - - - ``$MIRROR`` - - ``$PRIMARY`` - - ``$SECURITY`` - - ``$RELEASE`` - -**Internal name:** ``cc_apt_configure`` +import glob +import os +import re +from textwrap import dedent -**Module frequency:** per instance +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) +from cloudinit import gpg +from cloudinit import log as logging +from cloudinit import templater +from cloudinit import util +from cloudinit.settings import PER_INSTANCE -**Supported distros:** ubuntu, debian +LOG = logging.getLogger(__name__) -**Config keys**:: +# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') +ADD_APT_REPO_MATCH = r"^[\w-]+:\w" - apt: - preserve_sources_list: - disable_suites: +frequency = PER_INSTANCE +distros = ["ubuntu", "debian"] +mirror_property = { + 'type': 'array', + 'item': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['arches'], + 'properties': { + 'arches': { + 'type': 'array', + 'item': { + 'type': 'string' + }, + 'minItems': 1 + }, + 'uri': { + 'type': 'string', + 'format': 'uri' + }, + 'search': { + 'type': 'array', + 'item': { + 'type': 'string', + 'format': 'uri' + }, + 'minItems': 1 + }, + 'search_dns': { + 'type': 'boolean', + } + } + } +} +schema = { + 'id': 'cc_apt_configure', + 'name': 'Apt Configure', + 'title': 'Configure apt for the user', + 'description': dedent("""\ + This module handles both configuration of apt options and adding + source lists. There are configuration options such as + ``apt_get_wrapper`` and ``apt_get_command`` that control how + cloud-init invokes apt-get. These configuration options are + handled on a per-distro basis, so consult documentation for + cloud-init's distro support for instructions on using + these config options. + + .. note:: + To ensure that apt configuration is valid yaml, any strings + containing special characters, especially ``:`` should be quoted. + + .. note:: + For more information about apt configuration, see the + ``Additional apt configuration`` example."""), + 'distros': distros, + 'examples': [dedent("""\ + apt: + preserve_sources_list: false + disable_suites: - $RELEASE-updates - backports - $RELEASE - mysuite - primary: + primary: - arches: - amd64 - i386 - default - uri: "http://us.archive.ubuntu.com/ubuntu" + uri: 'http://us.archive.ubuntu.com/ubuntu' search: - - "http://cool.but-sometimes-unreachable.com/ubuntu" - - "http://us.archive.ubuntu.com/ubuntu" + - 'http://cool.but-sometimes-unreachable.com/ubuntu' + - 'http://us.archive.ubuntu.com/ubuntu' search_dns: - arches: - s390x - arm64 - uri: "http://archive-to-use-for-arm64.example.com/ubuntu" - security: + uri: 'http://archive-to-use-for-arm64.example.com/ubuntu' + security: - arches: - default search_dns: true - sources_list: | - deb $MIRROR $RELEASE main restricted - deb-src $MIRROR $RELEASE main restricted - deb $PRIMARY $RELEASE universe restricted - deb $SECURITY $RELEASE-security multiverse - debconf_selections: - set1: the-package the-package/some-flag boolean true - conf: | - APT { - Get { - Assume-Yes "true"; - Fix-Broken "true"; + sources_list: | + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + debconf_selections: + set1: the-package the-package/some-flag boolean true + conf: | + APT { + Get { + Assume-Yes 'true'; + Fix-Broken 'true'; + } + } + proxy: 'http://[[user][:pass]@]host[:port]/' + http_proxy: 'http://[[user][:pass]@]host[:port]/' + ftp_proxy: 'ftp://[[user][:pass]@]host[:port]/' + https_proxy: 'https://[[user][:pass]@]host[:port]/' + sources: + source1: + keyid: 'keyid' + keyserver: 'keyserverurl' + source: 'deb http:/// xenial main' + source2: + source: 'ppa:' + source3: + source: 'deb $MIRROR $RELEASE multiverse' + key: | + ------BEGIN PGP PUBLIC KEY BLOCK------- + + ------END PGP PUBLIC KEY BLOCK-------""")], + 'frequency': frequency, + 'type': 'object', + 'properties': { + 'apt': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'preserve_sources_list': { + 'type': 'boolean', + 'default': False, + 'description': dedent("""\ + By default, cloud-init will generate a new sources + list in ``/etc/apt/sources.list.d`` based on any + changes specified in cloud config. To disable this + behavior and preserve the sources list from the + pristine image, set ``preserve_sources_list`` + to ``true``. + + The ``preserve_sources_list`` option overrides + all other config keys that would alter + ``sources.list`` or ``sources.list.d``, + **except** for additional sources to be added + to ``sources.list.d``.""") + }, + 'disable_suites': { + 'type': 'array', + 'items': { + 'type': 'string' + }, + 'uniqueItems': True, + 'description': dedent("""\ + Entries in the sources list can be disabled using + ``disable_suites``, which takes a list of suites + to be disabled. If the string ``$RELEASE`` is + present in a suite in the ``disable_suites`` list, + it will be replaced with the release name. If a + suite specified in ``disable_suites`` is not + present in ``sources.list`` it will be ignored. + For convenience, several aliases are provided for + ``disable_suites``: + + - ``updates`` => ``$RELEASE-updates`` + - ``backports`` => ``$RELEASE-backports`` + - ``security`` => ``$RELEASE-security`` + - ``proposed`` => ``$RELEASE-proposed`` + - ``release`` => ``$RELEASE``. + + When a suite is disabled using ``disable_suites``, + its entry in ``sources.list`` is not deleted; it + is just commented out.""") + }, + 'primary': { + **mirror_property, + 'description': dedent("""\ + The primary and security archive mirrors can + be specified using the ``primary`` and + ``security`` keys, respectively. Both the + ``primary`` and ``security`` keys take a list + of configs, allowing mirrors to be specified + on a per-architecture basis. Each config is a + dictionary which must have an entry for + ``arches``, specifying which architectures + that config entry is for. The keyword + ``default`` applies to any architecture not + explicitly listed. The mirror url can be specified + with the ``uri`` key, or a list of mirrors to + check can be provided in order, with the first + mirror that can be resolved being selected. This + allows the same configuration to be used in + different environment, with different hosts used + for a local apt mirror. If no mirror is provided + by ``uri`` or ``search``, ``search_dns`` may be + used to search for dns names in the format + ``-mirror`` in each of the following: + + - fqdn of this host per cloud metadata, + - localdomain, + - domains listed in ``/etc/resolv.conf``. + + If there is a dns entry for ``-mirror``, + then it is assumed that there is a distro mirror + at ``http://-mirror./``. + If the ``primary`` key is defined, but not the + ``security`` key, then then configuration for + ``primary`` is also used for ``security``. + If ``search_dns`` is used for the ``security`` + key, the search pattern will be + ``-security-mirror``. + + If no mirrors are specified, or all lookups fail, + then default mirrors defined in the datasource + are used. If none are present in the datasource + either the following defaults are used: + + - ``primary`` => \ + ``http://archive.ubuntu.com/ubuntu``. + - ``security`` => \ + ``http://security.ubuntu.com/ubuntu`` + """)}, + 'security': { + **mirror_property, + 'description': dedent("""\ + Please refer to the primary config documentation""") + }, + 'add_apt_repo_match': { + 'type': 'string', + 'default': ADD_APT_REPO_MATCH, + 'description': dedent("""\ + All source entries in ``apt-sources`` that match + regex in ``add_apt_repo_match`` will be added to + the system using ``add-apt-repository``. If + ``add_apt_repo_match`` is not specified, it + defaults to ``{}``""".format(ADD_APT_REPO_MATCH)) + }, + 'debconf_selections': { + 'type': 'object', + 'items': {'type': 'string'}, + 'description': dedent("""\ + Debconf additional configurations can be specified as a + dictionary under the ``debconf_selections`` config + key, with each key in the dict representing a + different set of configurations. The value of each key + must be a string containing all the debconf + configurations that must be applied. We will bundle + all of the values and pass them to + ``debconf-set-selections``. Therefore, each value line + must be a valid entry for ``debconf-set-selections``, + meaning that they must possess for distinct fields: + + ``pkgname question type answer`` + + Where: + + - ``pkgname`` is the name of the package. + - ``question`` the name of the questions. + - ``type`` is the type of question. + - ``answer`` is the value used to ansert the \ + question. + + For example: \ + ``ippackage ippackage/ip string 127.0.01`` + """) + }, + 'sources_list': { + 'type': 'string', + 'description': dedent("""\ + Specifies a custom template for rendering + ``sources.list`` . If no ``sources_list`` template + is given, cloud-init will use sane default. Within + this template, the following strings will be + replaced with the appropriate values: + + - ``$MIRROR`` + - ``$RELEASE`` + - ``$PRIMARY`` + - ``$SECURITY``""") + }, + 'conf': { + 'type': 'string', + 'description': dedent("""\ + Specify configuration for apt, such as proxy + configuration. This configuration is specified as a + string. For multiline apt configuration, make sure + to follow yaml syntax.""") + }, + 'https_proxy': { + 'type': 'string', + 'description': dedent("""\ + More convenient way to specify https apt proxy. + https proxy url is specified in the format + ``https://[[user][:pass]@]host[:port]/``.""") + }, + 'http_proxy': { + 'type': 'string', + 'description': dedent("""\ + More convenient way to specify http apt proxy. + http proxy url is specified in the format + ``http://[[user][:pass]@]host[:port]/``.""") + }, + 'proxy': { + 'type': 'string', + 'description': 'Alias for defining a http apt proxy.' + }, + 'ftp_proxy': { + 'type': 'string', + 'description': dedent("""\ + More convenient way to specify ftp apt proxy. + ftp proxy url is specified in the format + ``ftp://[[user][:pass]@]host[:port]/``.""") + }, + 'sources': { + 'type': 'object', + 'items': {'type': 'string'}, + 'description': dedent("""\ + Source list entries can be specified as a + dictionary under the ``sources`` config key, with + each key in the dict representing a different source + file. The key of each source entry will be used + as an id that can be referenced in other config + entries, as well as the filename for the source's + configuration under ``/etc/apt/sources.list.d``. + If the name does not end with ``.list``, it will + be appended. If there is no configuration for a + key in ``sources``, no file will be written, but + the key may still be referred to as an id in other + ``sources`` entries. + + Each entry under ``sources`` is a dictionary which + may contain any of the following optional keys: + + - ``source``: a sources.list entry \ + (some variable replacements apply). + - ``keyid``: a key to import via shortid or \ + fingerprint. + - ``key``: a raw PGP key. + - ``keyserver``: alternate keyserver to pull \ + ``keyid`` key from. + + The ``source`` key supports variable + replacements for the following strings: + + - ``$MIRROR`` + - ``$PRIMARY`` + - ``$SECURITY`` + - ``$RELEASE``""") } } - proxy: "http://[[user][:pass]@]host[:port]/" - http_proxy: "http://[[user][:pass]@]host[:port]/" - ftp_proxy: "ftp://[[user][:pass]@]host[:port]/" - https_proxy: "https://[[user][:pass]@]host[:port]/" - sources: - source1: - keyid: "keyid" - keyserver: "keyserverurl" - source: "deb http:/// xenial main" - source2: - source: "ppa:" - source3: - source: "deb $MIRROR $RELEASE multiverse" - key: | - ------BEGIN PGP PUBLIC KEY BLOCK------- - - ------END PGP PUBLIC KEY BLOCK------- -""" - -import glob -import os -import re - -from cloudinit import gpg -from cloudinit import log as logging -from cloudinit import templater -from cloudinit import util + } + } +} -LOG = logging.getLogger(__name__) +__doc__ = get_schema_doc(schema) -# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') -ADD_APT_REPO_MATCH = r"^[\w-]+:\w" # place where apt stores cached repository data APT_LISTS = "/var/lib/apt/lists" @@ -279,6 +422,7 @@ def handle(name, ocfg, cloud, log, _): "Expected dictionary for 'apt' config, found {config_type}".format( config_type=type(cfg))) + validate_cloudconfig_schema(cfg, schema) apply_debconf_selections(cfg, target) apply_apt(cfg, cloud, target) diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 5498bbaa..3b2c2020 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -169,8 +169,8 @@ schema = { 'uniqueItems': True, 'description': dedent("""\ List of ntp pools. If both pools and servers are - empty, 4 default pool servers will be provided of - the format ``{0-3}.{distro}.pool.ntp.org``.""") + empty, 4 default pool servers will be provided of + the format ``{0-3}.{distro}.pool.ntp.org``.""") }, 'servers': { 'type': 'array', @@ -181,46 +181,46 @@ schema = { 'uniqueItems': True, 'description': dedent("""\ List of ntp servers. If both pools and servers are - empty, 4 default pool servers will be provided with - the format ``{0-3}.{distro}.pool.ntp.org``.""") + empty, 4 default pool servers will be provided with + the format ``{0-3}.{distro}.pool.ntp.org``.""") }, 'ntp_client': { 'type': 'string', 'default': 'auto', 'description': dedent("""\ Name of an NTP client to use to configure system NTP. - When unprovided or 'auto' the default client preferred - by the distribution will be used. The following - built-in client names can be used to override existing - configuration defaults: chrony, ntp, ntpdate, - systemd-timesyncd."""), + When unprovided or 'auto' the default client preferred + by the distribution will be used. The following + built-in client names can be used to override existing + configuration defaults: chrony, ntp, ntpdate, + systemd-timesyncd."""), }, 'enabled': { 'type': 'boolean', 'default': True, 'description': dedent("""\ Attempt to enable ntp clients if set to True. If set - to False, ntp client will not be configured or - installed"""), + to False, ntp client will not be configured or + installed"""), }, 'config': { 'description': dedent("""\ Configuration settings or overrides for the - ``ntp_client`` specified."""), + ``ntp_client`` specified."""), 'type': ['object'], 'properties': { 'confpath': { 'type': 'string', 'description': dedent("""\ The path to where the ``ntp_client`` - configuration is written."""), + configuration is written."""), }, 'check_exe': { 'type': 'string', 'description': dedent("""\ The executable name for the ``ntp_client``. - For example, ntp service ``check_exe`` is - 'ntpd' because it runs the ntpd binary."""), + For example, ntp service ``check_exe`` is + 'ntpd' because it runs the ntpd binary."""), }, 'packages': { 'type': 'array', @@ -230,22 +230,22 @@ schema = { 'uniqueItems': True, 'description': dedent("""\ List of packages needed to be installed for the - selected ``ntp_client``."""), + selected ``ntp_client``."""), }, 'service_name': { 'type': 'string', 'description': dedent("""\ The systemd or sysvinit service name used to - start and stop the ``ntp_client`` - service."""), + start and stop the ``ntp_client`` + service."""), }, 'template': { 'type': 'string', 'description': dedent("""\ Inline template allowing users to define their - own ``ntp_client`` configuration template. - The value must start with '## template:jinja' - to enable use of templating support. + own ``ntp_client`` configuration template. + The value must start with '## template:jinja' + to enable use of templating support. """), }, }, diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 204cbfd6..8601e707 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -103,7 +103,7 @@ schema = { 'type': 'string', 'description': dedent("""\ Path of the file to which ``content`` is decoded - and written + and written """), }, 'content': { @@ -111,9 +111,9 @@ schema = { 'default': '', 'description': dedent("""\ Optional content to write to the provided ``path``. - When content is present and encoding is not '%s', - decode the content prior to writing. Default: - **''** + When content is present and encoding is not '%s', + decode the content prior to writing. Default: + **''** """ % UNKNOWN_ENC), }, 'owner': { @@ -121,7 +121,7 @@ schema = { 'default': DEFAULT_OWNER, 'description': dedent("""\ Optional owner:group to chown on the file. Default: - **{owner}** + **{owner}** """.format(owner=DEFAULT_OWNER)), }, 'permissions': { @@ -129,8 +129,8 @@ schema = { 'default': oct(DEFAULT_PERMS).replace('o', ''), 'description': dedent("""\ Optional file permissions to set on ``path`` - represented as an octal string '0###'. Default: - **'{perms}'** + represented as an octal string '0###'. Default: + **'{perms}'** """.format(perms=oct(DEFAULT_PERMS).replace('o', ''))), }, 'encoding': { @@ -139,16 +139,16 @@ schema = { 'enum': supported_encoding_types, 'description': dedent("""\ Optional encoding type of the content. Default is - **text/plain** and no content decoding is - performed. Supported encoding types are: - %s.""" % ", ".join(supported_encoding_types)), + **text/plain** and no content decoding is + performed. Supported encoding types are: + %s.""" % ", ".join(supported_encoding_types)), }, 'append': { 'type': 'boolean', 'default': False, 'description': dedent("""\ Whether to append ``content`` to existing file if - ``path`` exists. Default: **false**. + ``path`` exists. Default: **false**. """), }, }, diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index a295d63d..da00060a 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -306,6 +306,28 @@ def _get_property_type(property_dict): return property_type +def _parse_description(description, prefix): + """Parse description from the schema in a format that we can better + display in our docs. This parser does three things: + + - Guarantee that a paragraph will be in a single line + - Guarantee that each new paragraph will be aligned with + the first paragraph + - Proper align lists of items + + @param description: The original description in the schema. + @param prefix: The number of spaces used to align the current description + """ + list_paragraph = prefix * 3 + description = re.sub(r"(\S)\n(\S)", r"\1 \2", description) + description = re.sub( + r"\n\n", r"\n\n{}".format(prefix), description) + description = re.sub( + r"\n( +)-", r"\n{}-".format(list_paragraph), description) + + return description + + def _get_property_doc(schema, prefix=' '): """Return restructured text describing the supported schema properties.""" new_prefix = prefix + ' ' @@ -318,7 +340,7 @@ def _get_property_doc(schema, prefix=' '): prefix=prefix, prop_name=prop_key, type=_get_property_type(prop_config), - description=description.replace('\n', ''))) + description=_parse_description(description, prefix))) items = prop_config.get('items') if items: if isinstance(items, list): diff --git a/doc/examples/cloud-config-apt.txt b/doc/examples/cloud-config-apt.txt index 5a56cd1b..004894b7 100644 --- a/doc/examples/cloud-config-apt.txt +++ b/doc/examples/cloud-config-apt.txt @@ -142,7 +142,7 @@ apt: # as above, allowing to have one config for different per arch mirrors # security is optional, if not defined it is set to the same value as primary security: - uri: http://security.ubuntu.com/ubuntu + - uri: http://security.ubuntu.com/ubuntu # If search_dns is set for security the searched pattern is: # -security-mirror diff --git a/doc/examples/cloud-config-chef-oneiric.txt b/doc/examples/cloud-config-chef-oneiric.txt index ab2e2a33..241fbf9b 100644 --- a/doc/examples/cloud-config-chef-oneiric.txt +++ b/doc/examples/cloud-config-chef-oneiric.txt @@ -13,38 +13,39 @@ # Key from http://apt.opscode.com/packages@opscode.com.gpg.key apt: sources: - - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1.4.9 (GNU/Linux) + source1: + source: "deb http://apt.opscode.com/ $RELEASE-0.10 main" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1.4.9 (GNU/Linux) - mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu - twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 - dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC - JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W - ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I - XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe - DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm - sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO - Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ - YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG - CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K - +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR - lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh - DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu - wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx - EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g - w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8 - AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN - QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X - Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ - 3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V - Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL - zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb - DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG - 0GLl8EkfA8uhluM= - =zKAm - -----END PGP PUBLIC KEY BLOCK----- + mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu + twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 + dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC + JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W + ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I + XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe + DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm + sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO + Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ + YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG + CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K + +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR + lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh + DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu + wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx + EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g + w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8 + AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN + QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X + Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ + 3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V + Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL + zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb + DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG + 0GLl8EkfA8uhluM= + =zKAm + -----END PGP PUBLIC KEY BLOCK----- chef: diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 8c1e4bb0..20a0ce0d 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -203,13 +203,14 @@ ssh_import_id: [smoser] # # Default: none # -debconf_selections: | # Need to preserve newlines +debconf_selections: # Force debconf priority to critical. - debconf debconf/priority select critical + set1: debconf debconf/priority select critical # Override default frontend to readline, but allow user to select. - debconf debconf/frontend select readline - debconf debconf/frontend seen false + set2: | + debconf debconf/frontend select readline + debconf debconf/frontend seen false # manage byobu defaults # byobu_by_default: diff --git a/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml b/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml index 0bec305e..68ca95b5 100644 --- a/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml +++ b/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml @@ -8,43 +8,44 @@ cloud_config: | #cloud-config # Key from https://packages.chef.io/chef.asc apt: - source1: - source: "deb http://packages.chef.io/repos/apt/stable $RELEASE main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1.4.12 (Darwin) - Comment: GPGTools - http://gpgtools.org + sources: + source1: + source: "deb http://packages.chef.io/repos/apt/stable $RELEASE main" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1.4.12 (Darwin) + Comment: GPGTools - http://gpgtools.org - mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu - twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 - dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC - JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W - ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I - XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe - DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm - sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO - Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ - YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG - CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K - +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu0IENIRUYgUGFja2FnZXMg - PHBhY2thZ2VzQGNoZWYuaW8+iGIEExECACIFAlQwYFECGwMGCwkIBwMCBhUIAgkK - CwQWAgMBAh4BAheAAAoJEClAq6mD74JqX94An26z99XOHWpLN8ahzm7cp13t4Xid - AJ9wVcgoUBzvgg91lKfv/34cmemZn7kCDQRKaQu0EAgAg7ZLCVGVTmLqBM6njZEd - Zbv+mZbvwLBSomdiqddE6u3eH0X3GuwaQfQWHUVG2yedyDMiG+EMtCdEeeRebTCz - SNXQ8Xvi22hRPoEsBSwWLZI8/XNg0n0f1+GEr+mOKO0BxDB2DG7DA0nnEISxwFkK - OFJFebR3fRsrWjj0KjDxkhse2ddU/jVz1BY7Nf8toZmwpBmdozETMOTx3LJy1HZ/ - Te9FJXJMUaB2lRyluv15MVWCKQJro4MQG/7QGcIfrIZNfAGJ32DDSjV7/YO+IpRY - IL4CUBQ65suY4gYUG4jhRH6u7H1p99sdwsg5OIpBe/v2Vbc/tbwAB+eJJAp89Zeu - twADBQf/ZcGoPhTGFuzbkcNRSIz+boaeWPoSxK2DyfScyCAuG41CY9+g0HIw9Sq8 - DuxQvJ+vrEJjNvNE3EAEdKl/zkXMZDb1EXjGwDi845TxEMhhD1dDw2qpHqnJ2mtE - WpZ7juGwA3sGhi6FapO04tIGacCfNNHmlRGipyq5ZiKIRq9mLEndlECr8cwaKgkS - 0wWu+xmMZe7N5/t/TK19HXNh4tVacv0F3fYK54GUjt2FjCQV75USnmNY4KPTYLXA - dzC364hEMlXpN21siIFgB04w+TXn5UF3B4FfAy5hevvr4DtV4MvMiGLu0oWjpaLC - MpmrR3Ny2wkmO0h+vgri9uIP06ODWIhJBBgRAgAJBQJKaQu0AhsMAAoJEClAq6mD - 74Jq4hIAoJ5KrYS8kCwj26SAGzglwggpvt3CAJ0bekyky56vNqoegB+y4PQVDv4K - zA== - =IxPr - -----END PGP PUBLIC KEY BLOCK----- + mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu + twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 + dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC + JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W + ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I + XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe + DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm + sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO + Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ + YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG + CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K + +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu0IENIRUYgUGFja2FnZXMg + PHBhY2thZ2VzQGNoZWYuaW8+iGIEExECACIFAlQwYFECGwMGCwkIBwMCBhUIAgkK + CwQWAgMBAh4BAheAAAoJEClAq6mD74JqX94An26z99XOHWpLN8ahzm7cp13t4Xid + AJ9wVcgoUBzvgg91lKfv/34cmemZn7kCDQRKaQu0EAgAg7ZLCVGVTmLqBM6njZEd + Zbv+mZbvwLBSomdiqddE6u3eH0X3GuwaQfQWHUVG2yedyDMiG+EMtCdEeeRebTCz + SNXQ8Xvi22hRPoEsBSwWLZI8/XNg0n0f1+GEr+mOKO0BxDB2DG7DA0nnEISxwFkK + OFJFebR3fRsrWjj0KjDxkhse2ddU/jVz1BY7Nf8toZmwpBmdozETMOTx3LJy1HZ/ + Te9FJXJMUaB2lRyluv15MVWCKQJro4MQG/7QGcIfrIZNfAGJ32DDSjV7/YO+IpRY + IL4CUBQ65suY4gYUG4jhRH6u7H1p99sdwsg5OIpBe/v2Vbc/tbwAB+eJJAp89Zeu + twADBQf/ZcGoPhTGFuzbkcNRSIz+boaeWPoSxK2DyfScyCAuG41CY9+g0HIw9Sq8 + DuxQvJ+vrEJjNvNE3EAEdKl/zkXMZDb1EXjGwDi845TxEMhhD1dDw2qpHqnJ2mtE + WpZ7juGwA3sGhi6FapO04tIGacCfNNHmlRGipyq5ZiKIRq9mLEndlECr8cwaKgkS + 0wWu+xmMZe7N5/t/TK19HXNh4tVacv0F3fYK54GUjt2FjCQV75USnmNY4KPTYLXA + dzC364hEMlXpN21siIFgB04w+TXn5UF3B4FfAy5hevvr4DtV4MvMiGLu0oWjpaLC + MpmrR3Ny2wkmO0h+vgri9uIP06ODWIhJBBgRAgAJBQJKaQu0AhsMAAoJEClAq6mD + 74Jq4hIAoJ5KrYS8kCwj26SAGzglwggpvt3CAJ0bekyky56vNqoegB+y4PQVDv4K + zA== + =IxPr + -----END PGP PUBLIC KEY BLOCK----- chef: diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 071b98bc..e19d13b8 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -24,6 +24,7 @@ class GetSchemaTest(CiTestCase): schema = get_schema() self.assertCountEqual( [ + 'cc_apt_configure', 'cc_bootcmd', 'cc_locale', 'cc_ntp', @@ -289,6 +290,41 @@ class GetSchemaDocTest(CiTestCase): """), get_schema_doc(full_schema)) + def test_get_schema_doc_properly_parse_description(self): + """get_schema_doc description properly formatted""" + full_schema = copy(self.required_schema) + full_schema.update( + {'properties': { + 'p1': { + 'type': 'string', + 'description': dedent("""\ + This item + has the + following options: + + - option1 + - option2 + - option3 + + The default value is + option1""") + } + }} + ) + + self.assertIn( + dedent(""" + **Config schema**: + **p1:** (string) This item has the following options: + + - option1 + - option2 + - option3 + + The default value is option1 + """), + get_schema_doc(full_schema)) + def test_get_schema_doc_raises_key_errors(self): """get_schema_doc raises KeyErrors on missing keys.""" for key in self.required_schema: -- cgit v1.2.3 From 4ab3303ec4bb49f029b7821d6dba53a6b02b6dc1 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Mon, 1 Jun 2020 14:20:39 -0700 Subject: test: fix all flake8 E741 errors (#401) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This removes the use of variables named ‘l’, ‘O’, or ‘I’. Generally these are used in list comprehension to read the line of lines. --- cloudinit/distros/__init__.py | 2 +- cloudinit/net/__init__.py | 8 ++++---- cloudinit/sources/DataSourceOpenNebula.py | 6 +++--- cloudinit/sources/helpers/openstack.py | 17 +++++++++++------ cloudinit/ssh_util.py | 4 +++- tests/cloud_tests/platforms/instances.py | 4 ++-- tests/cloud_tests/testcases/base.py | 4 ++-- tests/unittests/test_datasource/test_ec2.py | 4 +++- tests/unittests/test_datasource/test_ovf.py | 2 +- tests/unittests/test_ds_identify.py | 6 ++++-- .../test_handler/test_handler_apt_source_v3.py | 4 +++- tests/unittests/test_sshutil.py | 2 +- tox.ini | 3 +-- 13 files changed, 39 insertions(+), 27 deletions(-) (limited to 'tests/cloud_tests/testcases') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index e99529df..35a10590 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -582,7 +582,7 @@ class Distro(metaclass=abc.ABCMeta): # passwd must use short '-l' due to SLES11 lacking long form '--lock' lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name]) try: - cmd = next(l for l in lock_tools if util.which(l[0])) + cmd = next(tool for tool in lock_tools if util.which(tool[0])) except StopIteration: raise RuntimeError(( "Unable to lock user account '%s'. No tools available. " diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index cb8c1601..8af24fa9 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -824,13 +824,13 @@ def get_interfaces_by_mac_on_freebsd(): # flatten each interface block in a single line def flatten(out): curr_block = '' - for l in out.split('\n'): - if l.startswith('\t'): - curr_block += l + for line in out.split('\n'): + if line.startswith('\t'): + curr_block += line else: if curr_block: yield curr_block - curr_block = l + curr_block = line yield curr_block # looks for interface and mac in a list of flatten block diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 02c9a7b8..a08ab404 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -417,9 +417,9 @@ def read_context_disk_dir(source_dir, asuser=None): if ssh_key_var: lines = context.get(ssh_key_var).splitlines() - results['metadata']['public-keys'] = [l for l in lines - if len(l) and not - l.startswith("#")] + results['metadata']['public-keys'] = [ + line for line in lines if len(line) and not line.startswith("#") + ] # custom hostname -- try hostname or leave cloud-init # itself create hostname from IP address later diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index e91398ea..a4373f24 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -411,8 +411,11 @@ class ConfigDriveReader(BaseReader): keydata = meta_js.get('public-keys', keydata) if keydata: lines = keydata.splitlines() - md['public-keys'] = [l for l in lines - if len(l) and not l.startswith("#")] + md['public-keys'] = [ + line + for line in lines + if len(line) and not line.startswith("#") + ] # config-drive-v1 has no way for openstack to provide the instance-id # so we copy that into metadata from the user input @@ -674,11 +677,13 @@ def convert_net_json(network_json=None, known_macs=None): raise ValueError("Unable to find a system nic for %s" % d) d['name'] = known_macs[mac] - for cfg, key, fmt, target in link_updates: - if isinstance(target, (list, tuple)): - cfg[key] = [fmt % link_id_info[l]['name'] for l in target] + for cfg, key, fmt, targets in link_updates: + if isinstance(targets, (list, tuple)): + cfg[key] = [ + fmt % link_id_info[target]['name'] for target in targets + ] else: - cfg[key] = fmt % link_id_info[target]['name'] + cfg[key] = fmt % link_id_info[targets]['name'] # Infiniband interfaces may be referenced in network_data.json by a 6 byte # Ethernet MAC-style address, and we use that address to look up the diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index c3a9b5b7..918c4aec 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -344,7 +344,9 @@ def update_ssh_config(updates, fname=DEF_SSHD_CFG): changed = update_ssh_config_lines(lines=lines, updates=updates) if changed: util.write_file( - fname, "\n".join([str(l) for l in lines]) + "\n", copy_mode=True) + fname, "\n".join( + [str(line) for line in lines] + ) + "\n", copy_mode=True) return len(changed) != 0 diff --git a/tests/cloud_tests/platforms/instances.py b/tests/cloud_tests/platforms/instances.py index 529e79cd..efc35c7f 100644 --- a/tests/cloud_tests/platforms/instances.py +++ b/tests/cloud_tests/platforms/instances.py @@ -132,8 +132,8 @@ class Instance(TargetBase): """ def clean_test(test): """Clean formatting for system ready test testcase.""" - return ' '.join(l for l in test.strip().splitlines() - if not l.lstrip().startswith('#')) + return ' '.join(line for line in test.strip().splitlines() + if not line.lstrip().startswith('#')) boot_timeout = self.config['boot_timeout'] tests = [self.config['system_ready_script']] diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 7b67f54e..68d59111 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -141,8 +141,8 @@ class CloudTestCase(unittest.TestCase): def test_no_warnings_in_log(self): """Unexpected warnings should not be found in the log.""" warnings = [ - l for l in self.get_data_file('cloud-init.log').splitlines() - if 'WARN' in l] + line for line in self.get_data_file('cloud-init.log').splitlines() + if 'WARN' in line] joined_warnings = '\n'.join(warnings) for expected_warning in self.expected_warnings: self.assertIn( diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index d413d1cd..ad1ea595 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -612,7 +612,9 @@ class TestEc2(test_helpers.HttprettyTestCase): for log in expected_logs: self.assertIn(log, logs) self.assertEqual( - 1, len([l for l in logs.splitlines() if failed_put_log in l])) + 1, + len([line for line in logs.splitlines() if failed_put_log in line]) + ) def test_aws_token_redacted(self): """Verify that aws tokens are redacted when logged.""" diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index a19c35c8..486a2345 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -48,7 +48,7 @@ def fill_properties(props, template=OVF_ENV_CONTENT): for key, val in props.items(): lines.append(prop_tmpl.format(key=key, val=val)) indent = " " - properties = ''.join([indent + l + "\n" for l in lines]) + properties = ''.join([indent + line + "\n" for line in lines]) return template.format(properties=properties) diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index f0e96b44..c2318570 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -611,8 +611,10 @@ class TestDsIdentify(DsIdentifyBase): ret = self._check_via_dict( cust, RC_FOUND, func=".", args=[os.path.join(rootd, mpp)], rootd=rootd) - line = [l for l in ret.stdout.splitlines() if l.startswith(pre)][0] - toks = line.replace(pre, "").split(":") + match = [ + line for line in ret.stdout.splitlines() if line.startswith(pre) + ][0] + toks = match.replace(pre, "").split(":") expected = ["/sbin", "/bin", "/usr/sbin", "/usr/bin", "/mycust/path"] self.assertEqual(expected, [p for p in expected if p in toks], "path did not have expected tokens") diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py index 4762dbef..aefe26c4 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -1033,7 +1033,9 @@ class TestDebconfSelections(TestCase): # assumes called with *args value. selections = m_set_sel.call_args_list[0][0][0].decode() - missing = [l for l in lines if l not in selections.splitlines()] + missing = [ + line for line in lines if line not in selections.splitlines() + ] self.assertEqual([], missing) @mock.patch("cloudinit.config.cc_apt_configure.dpkg_reconfigure") diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index 0be41924..b4767f0c 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -299,7 +299,7 @@ class TestUpdateSshConfigLines(test_helpers.CiTestCase): lines = ssh_util.parse_ssh_config_lines(list(self.exlines)) result = ssh_util.update_ssh_config_lines(lines, updates) self.assertEqual([], result) - self.assertEqual(self.exlines, [str(l) for l in lines]) + self.assertEqual(self.exlines, [str(line) for line in lines]) def test_keycase_not_modified(self): """Original case of key should not be changed on update. diff --git a/tox.ini b/tox.ini index 611c3111..57b20568 100644 --- a/tox.ini +++ b/tox.ini @@ -49,10 +49,9 @@ deps = -r{toxinidir}/test-requirements.txt # E226: missing whitespace around arithmetic operator # E241: multiple spaces after ‘,’ # E402: module level import not at top of file -# E741: do not use variables named ‘l’, ‘O’, or ‘I’ # W503: line break before binary operator # W504: line break after binary operator -ignore=E121,E123,E126,E226,E241,E402,E741,W503,W504 +ignore=E121,E123,E126,E226,E241,E402,W503,W504 exclude = .venv,.tox,dist,doc,*egg,.git,build,tools [testenv:doc] -- cgit v1.2.3 From f3bd42659efeed4b092ffcdfd5df7f24813f2d3e Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Wed, 10 Jun 2020 07:39:29 -0700 Subject: test: fix all flake8 E126 errors (#425) --- cloudinit/cmd/devel/render.py | 5 +- cloudinit/cmd/query.py | 5 +- cloudinit/distros/netbsd.py | 10 +- cloudinit/distros/openbsd.py | 7 +- cloudinit/net/__init__.py | 5 +- cloudinit/net/bsd.py | 5 +- cloudinit/net/netbsd.py | 5 +- cloudinit/net/openbsd.py | 5 +- cloudinit/net/sysconfig.py | 2 +- cloudinit/reporting/handlers.py | 14 +- cloudinit/sources/DataSourceAzure.py | 33 +++-- cloudinit/sources/helpers/tests/test_netlink.py | 165 +++++++++++++-------- cloudinit/sources/tests/test_init.py | 9 +- tests/cloud_tests/testcases/base.py | 2 +- tests/unittests/test_datasource/test_azure.py | 33 +++-- tests/unittests/test_datasource/test_scaleway.py | 42 ++++-- tests/unittests/test_distros/test_bsd_utils.py | 5 +- .../unittests/test_handler/test_handler_mounts.py | 7 +- tests/unittests/test_reporting_hyperv.py | 11 +- tests/unittests/test_sshutil.py | 18 ++- tox.ini | 3 +- 21 files changed, 238 insertions(+), 153 deletions(-) (limited to 'tests/cloud_tests/testcases') diff --git a/cloudinit/cmd/devel/render.py b/cloudinit/cmd/devel/render.py index 1bc22406..1090aa16 100755 --- a/cloudinit/cmd/devel/render.py +++ b/cloudinit/cmd/devel/render.py @@ -57,8 +57,9 @@ def handle_args(name, args): paths.run_dir, INSTANCE_JSON_SENSITIVE_FILE) if not os.path.exists(instance_data_fn): LOG.warning( - 'Missing root-readable %s. Using redacted %s instead.', - instance_data_fn, redacted_data_fn) + 'Missing root-readable %s. Using redacted %s instead.', + instance_data_fn, redacted_data_fn + ) instance_data_fn = redacted_data_fn else: instance_data_fn = redacted_data_fn diff --git a/cloudinit/cmd/query.py b/cloudinit/cmd/query.py index e3db8679..0fb48ebd 100644 --- a/cloudinit/cmd/query.py +++ b/cloudinit/cmd/query.py @@ -90,8 +90,9 @@ def handle_args(name, args): instance_data_fn = sensitive_data_fn else: LOG.warning( - 'Missing root-readable %s. Using redacted %s instead.', - sensitive_data_fn, redacted_data_fn) + 'Missing root-readable %s. Using redacted %s instead.', + sensitive_data_fn, redacted_data_fn + ) instance_data_fn = redacted_data_fn else: instance_data_fn = redacted_data_fn diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py index 066737a8..f1a9b182 100644 --- a/cloudinit/distros/netbsd.py +++ b/cloudinit/distros/netbsd.py @@ -100,8 +100,9 @@ class NetBSD(cloudinit.distros.bsd.BSD): else: method = crypt.METHOD_BLOWFISH # pylint: disable=E1101 hashed_pw = crypt.crypt( - passwd, - crypt.mksalt(method)) + passwd, + crypt.mksalt(method) + ) try: subp.subp(['usermod', '-p', hashed_pw, user]) @@ -143,8 +144,9 @@ class NetBSD(cloudinit.distros.bsd.BSD): os_arch = platform.machine() e = os.environ.copy() e['PKG_PATH'] = ( - 'http://cdn.netbsd.org/pub/pkgsrc/' - 'packages/NetBSD/%s/%s/All') % (os_arch, os_release) + 'http://cdn.netbsd.org/pub/pkgsrc/' + 'packages/NetBSD/%s/%s/All' + ) % (os_arch, os_release) return e def update_package_sources(self): diff --git a/cloudinit/distros/openbsd.py b/cloudinit/distros/openbsd.py index 07c76530..720c9cf3 100644 --- a/cloudinit/distros/openbsd.py +++ b/cloudinit/distros/openbsd.py @@ -42,9 +42,10 @@ class Distro(cloudinit.distros.netbsd.NetBSD): os_arch = platform.machine() e = os.environ.copy() e['PKG_PATH'] = ( - 'ftp://ftp.openbsd.org/pub/OpenBSD/{os_release}/' - 'packages/{os_arch}/').format( - os_arch=os_arch, os_release=os_release) + 'ftp://ftp.openbsd.org/pub/OpenBSD/{os_release}/' + 'packages/{os_arch}/').format( + os_arch=os_arch, os_release=os_release + ) return e diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index a57fea0a..b40cb154 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -849,8 +849,9 @@ def get_interfaces_by_mac_on_freebsd(): def get_interfaces_by_mac_on_netbsd(): ret = {} re_field_match = ( - r"(?P\w+).*address:\s" - r"(?P([\da-f]{2}[:-]){5}([\da-f]{2})).*") + r"(?P\w+).*address:\s" + r"(?P([\da-f]{2}[:-]){5}([\da-f]{2})).*" + ) (out, _) = subp.subp(['ifconfig', '-a']) if_lines = re.sub(r'\n\s+', ' ', out).splitlines() for line in if_lines: diff --git a/cloudinit/net/bsd.py b/cloudinit/net/bsd.py index 1c355a98..e34e0454 100644 --- a/cloudinit/net/bsd.py +++ b/cloudinit/net/bsd.py @@ -66,8 +66,9 @@ class BSDRenderer(renderer.Renderer): if subnet.get('type') == 'static': if not subnet.get('netmask'): LOG.debug( - 'Skipping IP %s, because there is no netmask', - subnet.get('address')) + 'Skipping IP %s, because there is no netmask', + subnet.get('address') + ) continue LOG.debug('Configuring dev %s with %s / %s', device_name, subnet.get('address'), subnet.get('netmask')) diff --git a/cloudinit/net/netbsd.py b/cloudinit/net/netbsd.py index 30437b5f..71b38ee6 100644 --- a/cloudinit/net/netbsd.py +++ b/cloudinit/net/netbsd.py @@ -17,8 +17,9 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): if self.dhcp_interfaces(): self.set_rc_config_value('dhcpcd', 'YES') self.set_rc_config_value( - 'dhcpcd_flags', - ' '.join(self.dhcp_interfaces())) + 'dhcpcd_flags', + ' '.join(self.dhcp_interfaces()) + ) for device_name, v in self.interface_configurations.items(): if isinstance(v, dict): self.set_rc_config_value( diff --git a/cloudinit/net/openbsd.py b/cloudinit/net/openbsd.py index 489ea48b..166d77e6 100644 --- a/cloudinit/net/openbsd.py +++ b/cloudinit/net/openbsd.py @@ -19,8 +19,9 @@ class Renderer(cloudinit.net.bsd.BSDRenderer): elif isinstance(v, dict): try: content = "inet {address} {netmask}\n".format( - address=v['address'], - netmask=v['netmask']) + address=v['address'], + netmask=v['netmask'] + ) except KeyError: LOG.error( "Invalid static configuration for %s", diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index f36c300f..0a5d481d 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -505,7 +505,7 @@ class Renderer(renderer.Renderer): iface_cfg['IPADDR6_%d' % ipv6_index] = ipv6_cidr else: iface_cfg['IPV6ADDR_SECONDARIES'] += \ - " " + ipv6_cidr + " " + ipv6_cidr else: ipv4_index = ipv4_index + 1 suff = "" if ipv4_index == 0 else str(ipv4_index) diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 00e8d2e5..6b9127b6 100755 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -139,7 +139,8 @@ class HyperVKvpReportingHandler(ReportingHandler): self.event_key_prefix = u"{0}|{1}".format(self.EVENT_PREFIX, self.incarnation_no) self.publish_thread = threading.Thread( - target=self._publish_event_routine) + target=self._publish_event_routine + ) self.publish_thread.daemon = True self.publish_thread.start() @@ -202,10 +203,15 @@ class HyperVKvpReportingHandler(ReportingHandler): uuid.uuid4()) def _encode_kvp_item(self, key, value): - data = (struct.pack("%ds%ds" % ( + data = struct.pack( + "%ds%ds" + % ( self.HV_KVP_EXCHANGE_MAX_KEY_SIZE, - self.HV_KVP_EXCHANGE_MAX_VALUE_SIZE), - key.encode('utf-8'), value.encode('utf-8'))) + self.HV_KVP_EXCHANGE_MAX_VALUE_SIZE, + ), + key.encode("utf-8"), + value.encode("utf-8"), + ) return data def _decode_kvp_item(self, record_data): diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 89312b9e..6d569057 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -523,8 +523,9 @@ class DataSourceAzure(sources.DataSource): try: crawled_data = util.log_time( - logfunc=LOG.debug, msg='Crawl of metadata service', - func=self.crawl_metadata) + logfunc=LOG.debug, msg='Crawl of metadata service', + func=self.crawl_metadata + ) except sources.InvalidMetaDataException as e: LOG.warning('Could not crawl Azure metadata: %s', e) return False @@ -893,9 +894,10 @@ def can_dev_be_reformatted(devpath, preserve_ntfs): (cand_part, cand_path, devpath)) with events.ReportEventStack( - name="mount-ntfs-and-count", - description="mount-ntfs-and-count", - parent=azure_ds_reporter) as evt: + name="mount-ntfs-and-count", + description="mount-ntfs-and-count", + parent=azure_ds_reporter + ) as evt: try: file_count = util.mount_cb(cand_path, count_files, mtype="ntfs", update_env_for_mount={'LANG': 'C'}) @@ -924,9 +926,10 @@ def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, maxwait=120, # wait for ephemeral disk to come up naplen = .2 with events.ReportEventStack( - name="wait-for-ephemeral-disk", - description="wait for ephemeral disk", - parent=azure_ds_reporter): + name="wait-for-ephemeral-disk", + description="wait for ephemeral disk", + parent=azure_ds_reporter + ): missing = util.wait_for_files([devpath], maxwait=maxwait, naplen=naplen, @@ -1334,9 +1337,10 @@ def parse_network_config(imds_metadata): @return: Dictionary containing network version 2 standard configuration. """ with events.ReportEventStack( - name="parse_network_config", - description="", - parent=azure_ds_reporter) as evt: + name="parse_network_config", + description="", + parent=azure_ds_reporter + ) as evt: if imds_metadata != sources.UNSET and imds_metadata: netconfig = {'version': 2, 'ethernets': {}} LOG.debug('Azure: generating network configuration from IMDS') @@ -1480,9 +1484,10 @@ def maybe_remove_ubuntu_network_config_scripts(paths=None): def _is_platform_viable(seed_dir): with events.ReportEventStack( - name="check-platform-viability", - description="found azure asset tag", - parent=azure_ds_reporter) as evt: + name="check-platform-viability", + description="found azure asset tag", + parent=azure_ds_reporter + ) as evt: """Check platform environment to report if this datasource may run.""" asset_tag = util.read_dmi_data('chassis-asset-tag') diff --git a/cloudinit/sources/helpers/tests/test_netlink.py b/cloudinit/sources/helpers/tests/test_netlink.py index 58c3adc6..10760bd6 100644 --- a/cloudinit/sources/helpers/tests/test_netlink.py +++ b/cloudinit/sources/helpers/tests/test_netlink.py @@ -180,17 +180,22 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): other_ifname = "eth1" expected_ifname = "eth0" data_op_down_eth1 = self._media_switch_data( - other_ifname, RTM_NEWLINK, OPER_DOWN) + other_ifname, RTM_NEWLINK, OPER_DOWN + ) data_op_up_eth1 = self._media_switch_data( - other_ifname, RTM_NEWLINK, OPER_UP) + other_ifname, RTM_NEWLINK, OPER_UP + ) data_op_down_eth0 = self._media_switch_data( - expected_ifname, RTM_NEWLINK, OPER_DOWN) + expected_ifname, RTM_NEWLINK, OPER_DOWN + ) data_op_up_eth0 = self._media_switch_data( - expected_ifname, RTM_NEWLINK, OPER_UP) - m_read_netlink_socket.side_effect = [data_op_down_eth1, - data_op_up_eth1, - data_op_down_eth0, - data_op_up_eth0] + expected_ifname, RTM_NEWLINK, OPER_UP) + m_read_netlink_socket.side_effect = [ + data_op_down_eth1, + data_op_up_eth1, + data_op_down_eth0, + data_op_up_eth0 + ] wait_for_media_disconnect_connect(m_socket, expected_ifname) self.assertIn('Ignored netlink event on interface %s' % other_ifname, self.logs.getvalue()) @@ -207,17 +212,23 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): ''' ifname = "eth0" data_getlink_down = self._media_switch_data( - ifname, RTM_GETLINK, OPER_DOWN) + ifname, RTM_GETLINK, OPER_DOWN + ) data_getlink_up = self._media_switch_data( - ifname, RTM_GETLINK, OPER_UP) + ifname, RTM_GETLINK, OPER_UP + ) data_newlink_down = self._media_switch_data( - ifname, RTM_NEWLINK, OPER_DOWN) + ifname, RTM_NEWLINK, OPER_DOWN + ) data_newlink_up = self._media_switch_data( - ifname, RTM_NEWLINK, OPER_UP) - m_read_netlink_socket.side_effect = [data_getlink_down, - data_getlink_up, - data_newlink_down, - data_newlink_up] + ifname, RTM_NEWLINK, OPER_UP + ) + m_read_netlink_socket.side_effect = [ + data_getlink_down, + data_getlink_up, + data_newlink_down, + data_newlink_up + ] wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 4) @@ -233,19 +244,25 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): ''' ifname = "eth0" data_setlink_down = self._media_switch_data( - ifname, RTM_SETLINK, OPER_DOWN) + ifname, RTM_SETLINK, OPER_DOWN + ) data_setlink_up = self._media_switch_data( - ifname, RTM_SETLINK, OPER_UP) + ifname, RTM_SETLINK, OPER_UP + ) data_newlink_down = self._media_switch_data( - ifname, RTM_NEWLINK, OPER_DOWN) + ifname, RTM_NEWLINK, OPER_DOWN + ) data_newlink_up = self._media_switch_data( - ifname, RTM_NEWLINK, OPER_UP) - m_read_netlink_socket.side_effect = [data_setlink_down, - data_setlink_up, - data_newlink_down, - data_newlink_up, - data_newlink_down, - data_newlink_up] + ifname, RTM_NEWLINK, OPER_UP + ) + m_read_netlink_socket.side_effect = [ + data_setlink_down, + data_setlink_up, + data_newlink_down, + data_newlink_up, + data_newlink_down, + data_newlink_up + ] wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 4) @@ -255,23 +272,30 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): ifname = "eth0" data_op_down = self._media_switch_data(ifname, RTM_NEWLINK, OPER_DOWN) data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP) - data_op_dormant = self._media_switch_data(ifname, RTM_NEWLINK, - OPER_DORMANT) - data_op_notpresent = self._media_switch_data(ifname, RTM_NEWLINK, - OPER_NOTPRESENT) - data_op_lowerdown = self._media_switch_data(ifname, RTM_NEWLINK, - OPER_LOWERLAYERDOWN) - data_op_testing = self._media_switch_data(ifname, RTM_NEWLINK, - OPER_TESTING) - data_op_unknown = self._media_switch_data(ifname, RTM_NEWLINK, - OPER_UNKNOWN) - m_read_netlink_socket.side_effect = [data_op_up, data_op_up, - data_op_dormant, data_op_up, - data_op_notpresent, data_op_up, - data_op_lowerdown, data_op_up, - data_op_testing, data_op_up, - data_op_unknown, data_op_up, - data_op_down, data_op_up] + data_op_dormant = self._media_switch_data( + ifname, RTM_NEWLINK, OPER_DORMANT + ) + data_op_notpresent = self._media_switch_data( + ifname, RTM_NEWLINK, OPER_NOTPRESENT + ) + data_op_lowerdown = self._media_switch_data( + ifname, RTM_NEWLINK, OPER_LOWERLAYERDOWN + ) + data_op_testing = self._media_switch_data( + ifname, RTM_NEWLINK, OPER_TESTING + ) + data_op_unknown = self._media_switch_data( + ifname, RTM_NEWLINK, OPER_UNKNOWN + ) + m_read_netlink_socket.side_effect = [ + data_op_up, data_op_up, + data_op_dormant, data_op_up, + data_op_notpresent, data_op_up, + data_op_lowerdown, data_op_up, + data_op_testing, data_op_up, + data_op_unknown, data_op_up, + data_op_down, data_op_up + ] wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 14) @@ -281,12 +305,14 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): ifname = "eth0" data_op_down = self._media_switch_data(ifname, RTM_NEWLINK, OPER_DOWN) data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP) - data_op_dormant = self._media_switch_data(ifname, RTM_NEWLINK, - OPER_DORMANT) - data_op_unknown = self._media_switch_data(ifname, RTM_NEWLINK, - OPER_UNKNOWN) - m_read_netlink_socket.side_effect = [data_op_down, data_op_dormant, - data_op_unknown, data_op_up] + data_op_dormant = self._media_switch_data( + ifname, RTM_NEWLINK, OPER_DORMANT) + data_op_unknown = self._media_switch_data( + ifname, RTM_NEWLINK, OPER_UNKNOWN) + m_read_netlink_socket.side_effect = [ + data_op_down, data_op_dormant, + data_op_unknown, data_op_up + ] wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 4) @@ -300,9 +326,11 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): data_op_down = self._media_switch_data(ifname, RTM_NEWLINK, OPER_DOWN) data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP) data_op_invalid = self._media_switch_data(ifname, RTM_NEWLINK, 7) - m_read_netlink_socket.side_effect = [data_op_invalid, data_op_up, - data_op_down, data_op_invalid, - data_op_up] + m_read_netlink_socket.side_effect = [ + data_op_invalid, data_op_up, + data_op_down, data_op_invalid, + data_op_up + ] wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 5) @@ -333,8 +361,9 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): data_invalid2 = self._media_switch_data(ifname, RTM_NEWLINK, None) data_op_down = self._media_switch_data(ifname, RTM_NEWLINK, OPER_DOWN) data_op_up = self._media_switch_data(ifname, RTM_NEWLINK, OPER_UP) - m_read_netlink_socket.side_effect = [data_invalid1, data_invalid2, - data_op_down, data_op_up] + m_read_netlink_socket.side_effect = [ + data_invalid1, data_invalid2, data_op_down, data_op_up + ] wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 4) @@ -344,11 +373,15 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): bytes = ifname.encode("utf-8") data = bytearray(96) struct.pack_into("=LHHLL", data, 0, 48, RTM_NEWLINK, 0, 0, 0) - struct.pack_into("HH4sHHc", data, RTATTR_START_OFFSET, 8, 3, - bytes, 5, 16, int_to_bytes(OPER_DOWN)) + struct.pack_into( + "HH4sHHc", data, RTATTR_START_OFFSET, 8, 3, + bytes, 5, 16, int_to_bytes(OPER_DOWN) + ) struct.pack_into("=LHHLL", data, 48, 48, RTM_NEWLINK, 0, 0, 0) - struct.pack_into("HH4sHHc", data, 48 + RTATTR_START_OFFSET, 8, - 3, bytes, 5, 16, int_to_bytes(OPER_UP)) + struct.pack_into( + "HH4sHHc", data, 48 + RTATTR_START_OFFSET, 8, + 3, bytes, 5, 16, int_to_bytes(OPER_UP) + ) m_read_netlink_socket.return_value = data wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 1) @@ -360,14 +393,18 @@ class TestWaitForMediaDisconnectConnect(CiTestCase): data1 = bytearray(112) data2 = bytearray(32) struct.pack_into("=LHHLL", data1, 0, 48, RTM_NEWLINK, 0, 0, 0) - struct.pack_into("HH4sHHc", data1, RTATTR_START_OFFSET, 8, 3, - bytes, 5, 16, int_to_bytes(OPER_DOWN)) + struct.pack_into( + "HH4sHHc", data1, RTATTR_START_OFFSET, 8, 3, + bytes, 5, 16, int_to_bytes(OPER_DOWN) + ) struct.pack_into("=LHHLL", data1, 48, 48, RTM_NEWLINK, 0, 0, 0) - struct.pack_into("HH4sHHc", data1, 80, 8, 3, bytes, 5, 16, - int_to_bytes(OPER_DOWN)) + struct.pack_into( + "HH4sHHc", data1, 80, 8, 3, bytes, 5, 16, int_to_bytes(OPER_DOWN) + ) struct.pack_into("=LHHLL", data1, 96, 48, RTM_NEWLINK, 0, 0, 0) - struct.pack_into("HH4sHHc", data2, 16, 8, 3, bytes, 5, 16, - int_to_bytes(OPER_UP)) + struct.pack_into( + "HH4sHHc", data2, 16, 8, 3, bytes, 5, 16, int_to_bytes(OPER_UP) + ) m_read_netlink_socket.side_effect = [data1, data2] wait_for_media_disconnect_connect(m_socket, ifname) self.assertEqual(m_read_netlink_socket.call_count, 2) diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py index 5b6f1b3f..1420a988 100644 --- a/cloudinit/sources/tests/test_init.py +++ b/cloudinit/sources/tests/test_init.py @@ -436,10 +436,11 @@ class TestDataSource(CiTestCase): expected = { 'base64_encoded_keys': [], 'merged_cfg': { - '_doc': ( - 'Merged cloud-init system config from ' - '/etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/'), - 'datasource': {'_undef': {'key1': False}}}, + '_doc': ( + 'Merged cloud-init system config from ' + '/etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/' + ), + 'datasource': {'_undef': {'key1': False}}}, 'sensitive_keys': [ 'ds/meta_data/some/security-credentials', 'merged_cfg'], 'sys_info': sys_info, diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 68d59111..2e7c6686 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -321,7 +321,7 @@ class CloudTestCase(unittest.TestCase): "Unexpected sys_info dist value") self.assertEqual(self.os_name, v1_data['distro_release']) self.assertEqual( - str(self.os_cfg['version']), v1_data['distro_version']) + str(self.os_cfg['version']), v1_data['distro_version']) self.assertEqual('x86_64', v1_data['machine']) self.assertIsNotNone( re.match(r'3.\d\.\d', v1_data['python_version']), diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 05552a1e..a99cbd41 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -683,15 +683,17 @@ scbus-1 on xpt0 bus 0 'cloudinit.sources.DataSourceAzure.DataSourceAzure._report_ready') @mock.patch('cloudinit.sources.DataSourceAzure.DataSourceAzure._poll_imds') def test_crawl_metadata_on_reprovision_reports_ready( - self, poll_imds_func, - report_ready_func, - m_write, m_dhcp): + self, poll_imds_func, report_ready_func, m_write, m_dhcp + ): """If reprovisioning, report ready at the end""" ovfenv = construct_valid_ovf_env( - platform_settings={"PreprovisionedVm": "True"}) + platform_settings={"PreprovisionedVm": "True"} + ) - data = {'ovfcontent': ovfenv, - 'sys_cfg': {}} + data = { + 'ovfcontent': ovfenv, + 'sys_cfg': {} + } dsrc = self._get_ds(data) poll_imds_func.return_value = ovfenv dsrc.crawl_metadata() @@ -706,15 +708,18 @@ scbus-1 on xpt0 bus 0 @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') @mock.patch('cloudinit.sources.DataSourceAzure.readurl') def test_crawl_metadata_on_reprovision_reports_ready_using_lease( - self, m_readurl, m_dhcp, - m_net, report_ready_func, - m_media_switch, m_write): + self, m_readurl, m_dhcp, m_net, report_ready_func, + m_media_switch, m_write + ): """If reprovisioning, report ready using the obtained lease""" ovfenv = construct_valid_ovf_env( - platform_settings={"PreprovisionedVm": "True"}) + platform_settings={"PreprovisionedVm": "True"} + ) - data = {'ovfcontent': ovfenv, - 'sys_cfg': {}} + data = { + 'ovfcontent': ovfenv, + 'sys_cfg': {} + } dsrc = self._get_ds(data) lease = { @@ -1955,8 +1960,8 @@ class TestPreprovisioningPollIMDS(CiTestCase): response = requests.Response() response.status_code = 404 if self.tries == 2 else 410 raise requests.exceptions.HTTPError( - "fake {}".format(response.status_code), - response=response) + "fake {}".format(response.status_code), response=response + ) # Third try should succeed and stop retries or redhcp return mock.MagicMock(status_code=200, text="good", content="good") diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py index 15441454..9d82bda9 100644 --- a/tests/unittests/test_datasource/test_scaleway.py +++ b/tests/unittests/test_datasource/test_scaleway.py @@ -353,12 +353,16 @@ class TestDataSourceScaleway(HttprettyTestCase): self.datasource.metadata['ipv6'] = None netcfg = self.datasource.network_config - resp = {'version': 1, - 'config': [{ - 'type': 'physical', - 'name': 'ens2', - 'subnets': [{'type': 'dhcp4'}]}] + resp = { + 'version': 1, + 'config': [ + { + 'type': 'physical', + 'name': 'ens2', + 'subnets': [{'type': 'dhcp4'}] } + ] + } self.assertEqual(netcfg, resp) @mock.patch('cloudinit.sources.DataSourceScaleway.net.find_fallback_nic') @@ -424,12 +428,16 @@ class TestDataSourceScaleway(HttprettyTestCase): self.datasource.metadata['ipv6'] = None self.datasource._network_config = sources.UNSET - resp = {'version': 1, - 'config': [{ - 'type': 'physical', - 'name': 'ens2', - 'subnets': [{'type': 'dhcp4'}]}] + resp = { + 'version': 1, + 'config': [ + { + 'type': 'physical', + 'name': 'ens2', + 'subnets': [{'type': 'dhcp4'}] } + ] + } netcfg = self.datasource.network_config self.assertEqual(netcfg, resp) @@ -448,12 +456,16 @@ class TestDataSourceScaleway(HttprettyTestCase): self.datasource.metadata['ipv6'] = None self.datasource._network_config = None - resp = {'version': 1, - 'config': [{ - 'type': 'physical', - 'name': 'ens2', - 'subnets': [{'type': 'dhcp4'}]}] + resp = { + 'version': 1, + 'config': [ + { + 'type': 'physical', + 'name': 'ens2', + 'subnets': [{'type': 'dhcp4'}] } + ] + } netcfg = self.datasource.network_config self.assertEqual(netcfg, resp) diff --git a/tests/unittests/test_distros/test_bsd_utils.py b/tests/unittests/test_distros/test_bsd_utils.py index b38e4af5..3a68f2a9 100644 --- a/tests/unittests/test_distros/test_bsd_utils.py +++ b/tests/unittests/test_distros/test_bsd_utils.py @@ -62,5 +62,6 @@ class TestBsdUtils(CiTestCase): self.load_file.return_value = RC_FILE.format(hostname='foo') bsd_utils.set_rc_config_value('hostname', 'bar') self.write_file.assert_called_with( - '/etc/rc.conf', - RC_FILE.format(hostname='bar')) + '/etc/rc.conf', + RC_FILE.format(hostname='bar') + ) diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index 80c53c83..b643e3ae 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -260,8 +260,11 @@ class TestFstabHandling(test_helpers.FilesystemMockingTestCase): '/dev/vdb /mnt auto defaults,noexec,comment=cloudconfig 0 2\n' ) fstab_expected_content = fstab_original_content - cc = {'mounts': [ - ['/dev/vdb', '/mnt', 'auto', 'defaults,noexec']]} + cc = { + 'mounts': [ + ['/dev/vdb', '/mnt', 'auto', 'defaults,noexec'] + ] + } with open(cc_mounts.FSTAB_PATH, 'w') as fd: fd.write(fstab_original_content) with open(cc_mounts.FSTAB_PATH, 'r') as fd: diff --git a/tests/unittests/test_reporting_hyperv.py b/tests/unittests/test_reporting_hyperv.py index fa8f8859..b60a66ab 100644 --- a/tests/unittests/test_reporting_hyperv.py +++ b/tests/unittests/test_reporting_hyperv.py @@ -93,10 +93,15 @@ class TextKvpReporter(CiTestCase): def test_not_truncate_kvp_file_modified_after_boot(self): with open(self.tmp_file_path, "wb+") as f: kvp = {'key': 'key1', 'value': 'value1'} - data = (struct.pack("%ds%ds" % ( + data = struct.pack( + "%ds%ds" + % ( HyperVKvpReportingHandler.HV_KVP_EXCHANGE_MAX_KEY_SIZE, - HyperVKvpReportingHandler.HV_KVP_EXCHANGE_MAX_VALUE_SIZE), - kvp['key'].encode('utf-8'), kvp['value'].encode('utf-8'))) + HyperVKvpReportingHandler.HV_KVP_EXCHANGE_MAX_VALUE_SIZE, + ), + kvp["key"].encode("utf-8"), + kvp["value"].encode("utf-8"), + ) f.write(data) cur_time = time.time() os.utime(self.tmp_file_path, (cur_time, cur_time)) diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index b4767f0c..d15fc60b 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -374,13 +374,13 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase): sshd_config = self.tmp_path('sshd_config') util.write_file( - sshd_config, - "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys)) + sshd_config, + "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys) + ) (auth_key_fn, auth_key_entries) = ssh_util.extract_authorized_keys( - fpw.pw_name, sshd_config) - content = ssh_util.update_authorized_keys( - auth_key_entries, []) + fpw.pw_name, sshd_config) + content = ssh_util.update_authorized_keys(auth_key_entries, []) self.assertEqual("%s/.ssh/authorized_keys" % fpw.pw_dir, auth_key_fn) self.assertTrue(VALID_CONTENT['rsa'] in content) @@ -398,11 +398,13 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase): sshd_config = self.tmp_path('sshd_config') util.write_file( - sshd_config, - "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys)) + sshd_config, + "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys) + ) (auth_key_fn, auth_key_entries) = ssh_util.extract_authorized_keys( - fpw.pw_name, sshd_config) + fpw.pw_name, sshd_config + ) content = ssh_util.update_authorized_keys(auth_key_entries, []) self.assertEqual("%s/.ssh/authorized_keys" % fpw.pw_dir, auth_key_fn) diff --git a/tox.ini b/tox.ini index e04c7791..ebcebc41 100644 --- a/tox.ini +++ b/tox.ini @@ -43,11 +43,10 @@ basepython = python2.7 deps = -r{toxinidir}/test-requirements.txt [flake8] -# E126: continuation line over-indented for hanging indent # E226: missing whitespace around arithmetic operator # W503: line break before binary operator # W504: line break after binary operator -ignore=E126,E226,W503,W504 +ignore=E226,W503,W504 exclude = .venv,.tox,dist,doc,*egg,.git,build,tools per-file-ignores = cloudinit/cmd/main.py:E402 -- cgit v1.2.3 From 4fe576516d65feda17ba78e9265a8e494a195e7b Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 15 Jul 2020 10:26:12 -0400 Subject: cloudinit: remove global disable of pylint W0107 and fix errors (#489) * cloudinit: remove global disable of pylint W0107 and fix errors This includes removing a test class which contained no tests but wasn't detected as empty because of an errant pass statement. * .pylintrc: update disable comment to match arguments --- .pylintrc | 4 +--- cloudinit/config/cc_set_hostname.py | 1 - cloudinit/distros/networking.py | 3 --- cloudinit/distros/ubuntu.py | 2 -- cloudinit/net/cmdline.py | 2 -- cloudinit/net/dhcp.py | 2 -- cloudinit/reporting/handlers.py | 1 - cloudinit/sources/DataSourceAzure.py | 1 - cloudinit/sources/__init__.py | 2 -- cloudinit/sources/helpers/netlink.py | 1 - cloudinit/sources/helpers/vmware/imc/config_file.py | 1 - cloudinit/sources/helpers/vmware/imc/config_namespace.py | 1 - cloudinit/sources/helpers/vmware/imc/config_source.py | 1 - cloudinit/stages.py | 1 - tests/cloud_tests/platforms/azurecloud/instance.py | 2 -- tests/cloud_tests/platforms/images.py | 1 - tests/cloud_tests/platforms/snapshots.py | 1 - tests/cloud_tests/testcases/base.py | 1 - tests/unittests/test_datasource/test_ibmcloud.py | 7 ------- tests/unittests/test_reporting.py | 1 - 20 files changed, 1 insertion(+), 35 deletions(-) (limited to 'tests/cloud_tests/testcases') diff --git a/.pylintrc b/.pylintrc index f04603b9..94a81d0e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -7,7 +7,6 @@ jobs=4 [MESSAGES CONTROL] # Errors and warings with some filtered: -# W0107(unnecessary-pass) # W0201(attribute-defined-outside-init) # W0212(protected-access) # W0221(arguments-differ) @@ -19,7 +18,6 @@ jobs=4 # W0602(global-variable-not-assigned) # W0603(global-statement) # W0611(unused-import) -# W0612(unused-variable) # W0613(unused-argument) # W0621(redefined-outer-name) # W0622(redefined-builtin) @@ -27,7 +25,7 @@ jobs=4 # W0703(broad-except) # W1401(anomalous-backslash-in-string) -disable=C, F, I, R, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401 +disable=C, F, I, R, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401 [REPORTS] diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index 10d6d197..c13020d8 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -55,7 +55,6 @@ class SetHostnameError(Exception): This may happen if we attempt to set the hostname early in cloud-init's init-local timeframe as certain services may not be running yet. """ - pass def handle(name, cfg, cloud, log, _args): diff --git a/cloudinit/distros/networking.py b/cloudinit/distros/networking.py index 75e69df5..10ed249d 100644 --- a/cloudinit/distros/networking.py +++ b/cloudinit/distros/networking.py @@ -92,7 +92,6 @@ class Networking(metaclass=abc.ABCMeta): Examples of non-physical network devices: bonds, bridges, tunnels, loopback devices. """ - pass def is_renamed(self, devname: DeviceName) -> bool: return net.is_renamed(devname) @@ -117,7 +116,6 @@ class Networking(metaclass=abc.ABCMeta): process entirely, if the device already exists.) :type exists: Optional[DeviceName] """ - pass def wait_for_physdevs( self, netcfg: NetworkConfig, *, strict: bool = True @@ -182,7 +180,6 @@ class BSDNetworking(Networking): def settle(self, *, exists=None) -> None: """BSD has no equivalent to `udevadm settle`; noop.""" - pass class LinuxNetworking(Networking): diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index 23be3bdd..b4c4b0c3 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -49,7 +49,5 @@ class Distro(debian.Distro): copy.deepcopy(PREFERRED_NTP_CLIENTS)) return self._preferred_ntp_clients - pass - # vi: ts=4 expandtab diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 0e83685d..7ca7262b 100755 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -29,12 +29,10 @@ class InitramfsNetworkConfigSource(metaclass=abc.ABCMeta): @abc.abstractmethod def is_applicable(self) -> bool: """Is this initramfs config source applicable to the current system?""" - pass @abc.abstractmethod def render_config(self) -> dict: """Render a v1 network config from the initramfs configuration""" - pass class KlibcNetworkConfigSource(InitramfsNetworkConfigSource): diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 9e004688..d1d1255e 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -31,12 +31,10 @@ class InvalidDHCPLeaseFileError(Exception): Current uses are DataSourceAzure and DataSourceEc2 during ephemeral boot to scrape metadata. """ - pass class NoDHCPLeaseError(Exception): """Raised when unable to get a DHCP lease.""" - pass class EphemeralDHCPv4(object): diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 6b9127b6..1986ebdd 100755 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -35,7 +35,6 @@ class ReportingHandler(metaclass=abc.ABCMeta): def flush(self): """Ensure ReportingHandler has published all events""" - pass class LogHandler(ReportingHandler): diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 5e25b956..8b34d047 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -688,7 +688,6 @@ class DataSourceAzure(sources.DataSource): except UrlError: # Teardown our EphemeralDHCPv4 context on failure as we retry self._ephemeral_dhcp_ctx.clean_network() - pass finally: if nl_sock: nl_sock.close() diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 923e3cea..c4d60fff 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -78,7 +78,6 @@ class DataSourceNotFoundException(Exception): class InvalidMetaDataException(Exception): """Raised when metadata is broken, unavailable or disabled.""" - pass def process_instance_metadata(metadata, key_path='', sensitive_keys=()): @@ -511,7 +510,6 @@ class DataSource(metaclass=abc.ABCMeta): (e.g. 'ssh-rsa') and key_value is the key itself (e.g. 'AAAAB3NzaC1y...'). """ - pass def _remap_device(self, short_name): # LP: #611137 diff --git a/cloudinit/sources/helpers/netlink.py b/cloudinit/sources/helpers/netlink.py index d377ae3d..a74a3588 100644 --- a/cloudinit/sources/helpers/netlink.py +++ b/cloudinit/sources/helpers/netlink.py @@ -55,7 +55,6 @@ NetlinkHeader = namedtuple('NetlinkHeader', ['length', 'type', 'flags', 'seq', class NetlinkCreateSocketError(RuntimeError): '''Raised if netlink socket fails during create or bind.''' - pass def create_bound_netlink_socket(): diff --git a/cloudinit/sources/helpers/vmware/imc/config_file.py b/cloudinit/sources/helpers/vmware/imc/config_file.py index 602af078..fc034c95 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_file.py +++ b/cloudinit/sources/helpers/vmware/imc/config_file.py @@ -22,7 +22,6 @@ class ConfigFile(ConfigSource, dict): def __init__(self, filename): self._loadConfigFile(filename) - pass def _insertKey(self, key, val): """ diff --git a/cloudinit/sources/helpers/vmware/imc/config_namespace.py b/cloudinit/sources/helpers/vmware/imc/config_namespace.py index 2f29edd4..5899d8f7 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_namespace.py +++ b/cloudinit/sources/helpers/vmware/imc/config_namespace.py @@ -10,6 +10,5 @@ from .config_source import ConfigSource class ConfigNamespace(ConfigSource): """Specifies the Config Namespace.""" - pass # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py index 2f8ea546..7ec06a9c 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_source.py +++ b/cloudinit/sources/helpers/vmware/imc/config_source.py @@ -8,6 +8,5 @@ class ConfigSource(object): """Specifies a source for the Config Content.""" - pass # vi: ts=4 expandtab diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 69e6b7e1..765f4aab 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -947,7 +947,6 @@ def _pkl_load(fname): except Exception as e: if os.path.isfile(fname): LOG.warning("failed loading pickle in %s: %s", fname, e) - pass # This is allowed so just return nothing successfully loaded... if not pickle_contents: diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py index f1e28a96..a136cf0d 100644 --- a/tests/cloud_tests/platforms/azurecloud/instance.py +++ b/tests/cloud_tests/platforms/azurecloud/instance.py @@ -80,7 +80,6 @@ class AzureCloudInstance(Instance): except CloudError: LOG.debug(('image not found, launching instance with base image, ' 'image_id=%s'), self.image_id) - pass vm_params = { 'name': self.vm_name, @@ -169,7 +168,6 @@ class AzureCloudInstance(Instance): sleep(15) else: LOG.warning('Could not find console log: %s', e) - pass LOG.debug('stopping instance %s', self.image_id) vm_deallocate = \ diff --git a/tests/cloud_tests/platforms/images.py b/tests/cloud_tests/platforms/images.py index 557a5cf6..f047de2e 100644 --- a/tests/cloud_tests/platforms/images.py +++ b/tests/cloud_tests/platforms/images.py @@ -52,6 +52,5 @@ class Image(TargetBase): def destroy(self): """Clean up data associated with image.""" - pass # vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms/snapshots.py b/tests/cloud_tests/platforms/snapshots.py index 94328982..0f5f8bb6 100644 --- a/tests/cloud_tests/platforms/snapshots.py +++ b/tests/cloud_tests/platforms/snapshots.py @@ -40,6 +40,5 @@ class Snapshot(object): def destroy(self): """Clean up snapshot data.""" - pass # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 2e7c6686..4448e0b5 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -34,7 +34,6 @@ class CloudTestCase(unittest.TestCase): @classmethod def maybeSkipTest(cls): """Present to allow subclasses to override and raise a skipTest.""" - pass def assertPackageInstalled(self, name, version=None): """Check dpkg-query --show output for matching package name. diff --git a/tests/unittests/test_datasource/test_ibmcloud.py b/tests/unittests/test_datasource/test_ibmcloud.py index 0b54f585..9013ae9f 100644 --- a/tests/unittests/test_datasource/test_ibmcloud.py +++ b/tests/unittests/test_datasource/test_ibmcloud.py @@ -15,13 +15,6 @@ mock = test_helpers.mock D_PATH = "cloudinit.sources.DataSourceIBMCloud." -class TestIBMCloud(test_helpers.CiTestCase): - """Test the datasource.""" - def setUp(self): - super(TestIBMCloud, self).setUp() - pass - - @mock.patch(D_PATH + "_is_xen", return_value=True) @mock.patch(D_PATH + "_is_ibm_provisioning") @mock.patch(D_PATH + "util.blkid") diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 6814030e..9f11fd5c 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -349,7 +349,6 @@ class TestReportingEventStack(TestCase): with parent: with child: pass - pass self.assertEqual(report_start.call_count, 0) self.assertEqual(report_finish.call_count, 0) -- cgit v1.2.3 From 07104504ab5b30efd2d1f7a8c36effe18b8e5fe0 Mon Sep 17 00:00:00 2001 From: Paride Legovini Date: Tue, 25 Aug 2020 17:21:18 +0200 Subject: tox: bump the pylint version to 2.6.0 in the default run (#544) Changes: tox: bump the pylint version to 2.6.0 in the default run Fix pylint 2.6.0 W0707 warnings (raise-missing-from) --- cloudinit/analyze/show.py | 2 +- cloudinit/cmd/devel/make_mime.py | 6 ++-- cloudinit/config/cc_disk_setup.py | 36 ++++++++++++++-------- cloudinit/config/cc_growpart.py | 8 ++--- cloudinit/config/cc_power_state_change.py | 12 +++++--- cloudinit/config/cc_rsyslog.py | 6 ++-- cloudinit/config/cc_set_hostname.py | 2 +- cloudinit/config/cc_ubuntu_advantage.py | 2 +- cloudinit/config/schema.py | 2 +- cloudinit/distros/__init__.py | 5 +-- cloudinit/distros/arch.py | 4 +-- cloudinit/distros/parsers/resolv_conf.py | 7 +++-- cloudinit/gpg.py | 5 +-- cloudinit/handlers/jinja_template.py | 3 +- cloudinit/net/__init__.py | 7 +++-- cloudinit/net/cmdline.py | 4 +-- cloudinit/net/dhcp.py | 4 +-- cloudinit/net/network_state.py | 19 +++++++----- cloudinit/sources/DataSourceAzure.py | 2 +- cloudinit/sources/DataSourceEc2.py | 8 +++-- cloudinit/sources/DataSourceIBMCloud.py | 3 +- cloudinit/sources/DataSourceMAAS.py | 3 +- cloudinit/sources/DataSourceOpenNebula.py | 13 +++++--- cloudinit/sources/DataSourceOpenStack.py | 4 +-- cloudinit/sources/DataSourceSmartOS.py | 4 ++- cloudinit/sources/helpers/azure.py | 34 +++++++++++--------- cloudinit/sources/helpers/netlink.py | 2 +- cloudinit/sources/helpers/openstack.py | 34 +++++++++++--------- cloudinit/subp.py | 3 +- cloudinit/url_helper.py | 6 ++-- cloudinit/util.py | 8 ++--- tests/cloud_tests/platforms/azurecloud/instance.py | 7 +++-- tests/cloud_tests/platforms/azurecloud/platform.py | 23 +++++++++----- tests/cloud_tests/platforms/ec2/instance.py | 4 +-- tests/cloud_tests/platforms/ec2/platform.py | 18 ++++++----- tests/cloud_tests/platforms/lxd/instance.py | 3 +- tests/cloud_tests/platforms/platforms.py | 6 ++-- tests/cloud_tests/testcases/__init__.py | 6 ++-- tools/mock-meta.py | 14 +++++---- tox.ini | 2 +- 40 files changed, 205 insertions(+), 136 deletions(-) (limited to 'tests/cloud_tests/testcases') diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py index 0c825b23..01a4d3e5 100644 --- a/cloudinit/analyze/show.py +++ b/cloudinit/analyze/show.py @@ -267,7 +267,7 @@ def gather_timestamps_using_systemd(): except OSError as err: raise RuntimeError('Could not determine container boot ' 'time from /proc/1/cmdline. ({})' - .format(err)) + .format(err)) from err status = CONTAINER_CODE else: status = FAIL_CODE diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py index 77e10540..4e6a5778 100755 --- a/cloudinit/cmd/devel/make_mime.py +++ b/cloudinit/cmd/devel/make_mime.py @@ -22,8 +22,10 @@ def file_content_type(text): try: filename, content_type = text.split(":", 1) return (open(filename, 'r'), filename, content_type.strip()) - except ValueError: - raise argparse.ArgumentError(text, "Invalid value for %r" % (text)) + except ValueError as e: + raise argparse.ArgumentError( + text, "Invalid value for %r" % (text) + ) from e def get_parser(parser=None): diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index d957cfe3..a7bdc703 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -251,7 +251,9 @@ def enumerate_disk(device, nodeps=False): try: info, _err = subp.subp(lsblk_cmd) except Exception as e: - raise Exception("Failed during disk check for %s\n%s" % (device, e)) + raise Exception( + "Failed during disk check for %s\n%s" % (device, e) + ) from e parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0] @@ -313,7 +315,9 @@ def check_fs(device): try: out, _err = subp.subp(blkid_cmd, rcs=[0, 2]) except Exception as e: - raise Exception("Failed during disk check for %s\n%s" % (device, e)) + raise Exception( + "Failed during disk check for %s\n%s" % (device, e) + ) from e if out: if len(out.splitlines()) == 1: @@ -428,8 +432,8 @@ def get_dyn_func(*args): else: return globals()[func_name] - except KeyError: - raise Exception("No such function %s to call!" % func_name) + except KeyError as e: + raise Exception("No such function %s to call!" % func_name) from e def get_hdd_size(device): @@ -437,7 +441,7 @@ def get_hdd_size(device): size_in_bytes, _ = subp.subp([BLKDEV_CMD, '--getsize64', device]) sector_size, _ = subp.subp([BLKDEV_CMD, '--getss', device]) except Exception as e: - raise Exception("Failed to get %s size\n%s" % (device, e)) + raise Exception("Failed to get %s size\n%s" % (device, e)) from e return int(size_in_bytes) / int(sector_size) @@ -455,8 +459,9 @@ def check_partition_mbr_layout(device, layout): try: out, _err = subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: - raise Exception("Error running partition command on %s\n%s" % ( - device, e)) + raise Exception( + "Error running partition command on %s\n%s" % (device, e) + ) from e found_layout = [] for line in out.splitlines(): @@ -485,8 +490,9 @@ def check_partition_gpt_layout(device, layout): try: out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV) except Exception as e: - raise Exception("Error running partition command on %s\n%s" % ( - device, e)) + raise Exception( + "Error running partition command on %s\n%s" % (device, e) + ) from e out_lines = iter(out.splitlines()) # Skip header. Output looks like: @@ -657,8 +663,10 @@ def purge_disk(device): try: LOG.info("Purging filesystem on /dev/%s", d['name']) subp.subp(wipefs_cmd) - except Exception: - raise Exception("Failed FS purge of /dev/%s" % d['name']) + except Exception as e: + raise Exception( + "Failed FS purge of /dev/%s" % d['name'] + ) from e purge_disk_ptable(device) @@ -700,7 +708,9 @@ def exec_mkpart_mbr(device, layout): try: subp.subp(prt_cmd, data="%s\n" % layout) except Exception as e: - raise Exception("Failed to partition device %s\n%s" % (device, e)) + raise Exception( + "Failed to partition device %s\n%s" % (device, e) + ) from e read_parttbl(device) @@ -997,6 +1007,6 @@ def mkfs(fs_cfg): try: subp.subp(fs_cmd, shell=shell) except Exception as e: - raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) + raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) from e # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index c5d93f81..237c3d02 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -148,14 +148,14 @@ class ResizeGrowPart(object): if e.exit_code != 1: util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)", diskdev, partnum) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e return (before, before) try: subp.subp(["growpart", diskdev, partnum]) except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: growpart %s %s", diskdev, partnum) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e return (before, get_size(partdev)) @@ -187,14 +187,14 @@ class ResizeGpart(object): except subp.ProcessExecutionError as e: if e.exit_code != 0: util.logexc(LOG, "Failed: gpart recover %s", diskdev) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e before = get_size(partdev) try: subp.subp(["gpart", "resize", "-i", partnum, diskdev]) except subp.ProcessExecutionError as e: util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev) - raise ResizeFailedException(e) + raise ResizeFailedException(e) from e # Since growing the FS requires a reboot, make sure we reboot # first when this module has finished. diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index ab953a0d..6fcb8a7d 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -196,10 +196,11 @@ def load_power_state(cfg, distro_name): try: delay = convert_delay(delay, fmt=fmt, scale=scale) - except ValueError: + except ValueError as e: raise TypeError( "power_state[delay] must be 'now' or '+m' (minutes)." - " found '%s'." % delay) + " found '%s'." % delay + ) from e args = command + [delay] if message: @@ -207,9 +208,10 @@ def load_power_state(cfg, distro_name): try: timeout = float(pstate.get('timeout', 30.0)) - except ValueError: - raise ValueError("failed to convert timeout '%s' to float." % - pstate['timeout']) + except ValueError as e: + raise ValueError( + "failed to convert timeout '%s' to float." % pstate['timeout'] + ) from e condition = pstate.get("condition", True) if not isinstance(condition, (str, list, bool)): diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 1354885a..2a2bc931 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -347,8 +347,10 @@ class SyslogRemotesLine(object): if self.port: try: int(self.port) - except ValueError: - raise ValueError("port '%s' is not an integer" % self.port) + except ValueError as e: + raise ValueError( + "port '%s' is not an integer" % self.port + ) from e if not self.addr: raise ValueError("address is required") diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index c13020d8..1d23d80d 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -85,7 +85,7 @@ def handle(name, cfg, cloud, log, _args): except Exception as e: msg = "Failed to set the hostname to %s (%s)" % (fqdn, hostname) util.logexc(log, msg) - raise SetHostnameError("%s: %s" % (msg, e)) + raise SetHostnameError("%s: %s" % (msg, e)) from e write_json(prev_fn, {'hostname': hostname, 'fqdn': fqdn}) # vi: ts=4 expandtab diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py index 35ded5db..d61dc655 100644 --- a/cloudinit/config/cc_ubuntu_advantage.py +++ b/cloudinit/config/cc_ubuntu_advantage.py @@ -115,7 +115,7 @@ def configure_ua(token=None, enable=None): msg = 'Failure attaching Ubuntu Advantage:\n{error}'.format( error=str(e)) util.logexc(LOG, msg) - raise RuntimeError(msg) + raise RuntimeError(msg) from e enable_errors = [] for service in enable: try: diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 2d8c7577..8a966aee 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -210,7 +210,7 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): error = SchemaValidationError(errors) if annotate: print(annotated_cloudconfig_file({}, content, error.schema_errors)) - raise error + raise error from e try: validate_cloudconfig_schema( cloudconfig, schema, strict=True) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index effb4276..2537608f 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -607,10 +607,11 @@ class Distro(metaclass=abc.ABCMeta): lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name]) try: cmd = next(tool for tool in lock_tools if subp.which(tool[0])) - except StopIteration: + except StopIteration as e: raise RuntimeError(( "Unable to lock user account '%s'. No tools available. " - " Tried: %s.") % (name, [c[0] for c in lock_tools])) + " Tried: %s.") % (name, [c[0] for c in lock_tools]) + ) from e try: subp.subp(cmd) except Exception as e: diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 038aa9ac..967be168 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -61,9 +61,9 @@ class Distro(distros.Distro): def _write_network_config(self, netconfig): try: return self._supported_write_network_config(netconfig) - except RendererNotFoundError: + except RendererNotFoundError as e: # Fall back to old _write_network - raise NotImplementedError + raise NotImplementedError from e def _write_network(self, settings): entries = net_util.translate_network(settings) diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py index 299d54b5..62929d03 100644 --- a/cloudinit/distros/parsers/resolv_conf.py +++ b/cloudinit/distros/parsers/resolv_conf.py @@ -150,9 +150,10 @@ class ResolvConf(object): tail = '' try: (cfg_opt, cfg_values) = head.split(None, 1) - except (IndexError, ValueError): - raise IOError("Incorrectly formatted resolv.conf line %s" - % (i + 1)) + except (IndexError, ValueError) as e: + raise IOError( + "Incorrectly formatted resolv.conf line %s" % (i + 1) + ) from e if cfg_opt not in ['nameserver', 'domain', 'search', 'sortlist', 'options']: raise IOError("Unexpected resolv.conf option %s" % (cfg_opt)) diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py index 72b5ac59..be0ca0ea 100644 --- a/cloudinit/gpg.py +++ b/cloudinit/gpg.py @@ -63,10 +63,11 @@ def recv_key(key, keyserver, retries=(1, 1)): "Import failed with exit code %d, will try again in %ss", error.exit_code, naplen) time.sleep(naplen) - except StopIteration: + except StopIteration as e: raise ValueError( ("Failed to import key '%s' from keyserver '%s' " - "after %d tries: %s") % (key, keyserver, trynum, error)) + "after %d tries: %s") % (key, keyserver, trynum, error) + ) from e def delete_key(key): diff --git a/cloudinit/handlers/jinja_template.py b/cloudinit/handlers/jinja_template.py index ce3accf6..aadfbf86 100644 --- a/cloudinit/handlers/jinja_template.py +++ b/cloudinit/handlers/jinja_template.py @@ -83,7 +83,8 @@ def render_jinja_payload_from_file( if e.errno == EACCES: raise RuntimeError( 'Cannot render jinja template vars. No read permission on' - " '%s'. Try sudo" % instance_data_file) + " '%s'. Try sudo" % instance_data_file + ) from e rendered_payload = render_jinja_payload( payload, payload_fn, instance_data, debug) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 322af77b..e233149a 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -506,7 +506,9 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): try: _rename_interfaces(extract_physdevs(netcfg)) except RuntimeError as e: - raise RuntimeError('Failed to apply network config names: %s' % e) + raise RuntimeError( + 'Failed to apply network config names: %s' % e + ) from e def interface_has_own_mac(ifname, strict=False): @@ -965,7 +967,8 @@ class EphemeralIPv4Network(object): self.prefix = mask_to_net_prefix(prefix_or_mask) except ValueError as e: raise ValueError( - 'Cannot setup network: {0}'.format(e)) + 'Cannot setup network: {0}'.format(e) + ) from e self.connectivity_url = connectivity_url self.interface = interface diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 7ca7262b..cc8dc17b 100755 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -112,8 +112,8 @@ def _klibc_to_config_entry(content, mac_addrs=None): data = util.load_shell_content(content) try: name = data['DEVICE'] if 'DEVICE' in data else data['DEVICE6'] - except KeyError: - raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data") + except KeyError as e: + raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data") from e # ipconfig on precise does not write PROTO # IPv6 config gives us IPV6PROTO, not PROTO. diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 9230cf7a..4394c68b 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -82,8 +82,8 @@ class EphemeralDHCPv4(object): try: leases = maybe_perform_dhcp_discovery( self.iface, self.dhcp_log_func) - except InvalidDHCPLeaseFileError: - raise NoDHCPLeaseError() + except InvalidDHCPLeaseFileError as e: + raise NoDHCPLeaseError() from e if not leases: raise NoDHCPLeaseError() self.lease = leases[-1] diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 7bfe8be0..b2f7d31e 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -297,9 +297,10 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): command_type = command['type'] try: handler = self.command_handlers[command_type] - except KeyError: - raise RuntimeError("No handler found for" - " command '%s'" % command_type) + except KeyError as e: + raise RuntimeError( + "No handler found for command '%s'" % command_type + ) from e try: handler(self, command) except InvalidCommand: @@ -316,9 +317,10 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): continue try: handler = self.command_handlers[command_type] - except KeyError: - raise RuntimeError("No handler found for" - " command '%s'" % command_type) + except KeyError as e: + raise RuntimeError( + "No handler found for command '%s'" % command_type + ) from e try: handler(self, command) self._v2_common(command) @@ -914,9 +916,10 @@ def _normalize_route(route): if metric: try: normal_route['metric'] = int(metric) - except ValueError: + except ValueError as e: raise TypeError( - 'Route config metric {} is not an integer'.format(metric)) + 'Route config metric {} is not an integer'.format(metric) + ) from e return normal_route diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 86cc7c28..f3c6452b 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -1147,7 +1147,7 @@ def read_azure_ovf(contents): except Exception as e: error_str = "Invalid ovf-env.xml: %s" % e report_diagnostic_event(error_str) - raise BrokenAzureDataSource(error_str) + raise BrokenAzureDataSource(error_str) from e results = find_child(dom.documentElement, lambda n: n.localName == "ProvisioningSection") diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 355b4e2f..1d09c12a 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -617,9 +617,11 @@ def parse_strict_mode(cfgval): if sleep: try: sleep = int(sleep) - except ValueError: - raise ValueError("Invalid sleep '%s' in strict_id setting '%s': " - "not an integer" % (sleep, cfgval)) + except ValueError as e: + raise ValueError( + "Invalid sleep '%s' in strict_id setting '%s': not an integer" + % (sleep, cfgval) + ) from e else: sleep = None diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py index d2aa9a58..8d196185 100644 --- a/cloudinit/sources/DataSourceIBMCloud.py +++ b/cloudinit/sources/DataSourceIBMCloud.py @@ -303,7 +303,8 @@ def read_md(): except sources.BrokenMetadata as e: raise RuntimeError( "Failed reading IBM config disk (platform=%s path=%s): %s" % - (platform, path, e)) + (platform, path, e) + ) from e ret.update(results) return ret diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index c80f70c2..9156925f 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -226,7 +226,8 @@ def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None, except url_helper.UrlError as e: if e.code == 404 and not optional: raise MAASSeedDirMalformed( - "Missing required %s: %s" % (path, e)) + "Missing required %s: %s" % (path, e) + ) from e elif e.code != 404: raise e diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 12b1f94f..45481938 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -399,18 +399,23 @@ def read_context_disk_dir(source_dir, distro, asuser=None): if asuser is not None: try: pwd.getpwnam(asuser) - except KeyError: + except KeyError as e: raise BrokenContextDiskDir( "configured user '{user}' does not exist".format( - user=asuser)) + user=asuser) + ) from e try: path = os.path.join(source_dir, 'context.sh') content = util.load_file(path) context = parse_shell_config(content, asuser=asuser) except subp.ProcessExecutionError as e: - raise BrokenContextDiskDir("Error processing context.sh: %s" % (e)) + raise BrokenContextDiskDir( + "Error processing context.sh: %s" % (e) + ) from e except IOError as e: - raise NonContextDiskDir("Error reading context.sh: %s" % (e)) + raise NonContextDiskDir( + "Error reading context.sh: %s" % (e) + ) from e else: raise NonContextDiskDir("Missing context.sh") diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index bf539091..d4b43f44 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -194,10 +194,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'timeout': url_params.timeout_seconds}) except openstack.NonReadable as e: raise sources.InvalidMetaDataException(str(e)) - except (openstack.BrokenMetadata, IOError): + except (openstack.BrokenMetadata, IOError) as e: msg = 'Broken metadata address {addr}'.format( addr=self.metadata_address) - raise sources.InvalidMetaDataException(msg) + raise sources.InvalidMetaDataException(msg) from e return result diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 843b3a2a..f1f903bc 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -413,7 +413,9 @@ class JoyentMetadataClient(object): response.append(byte) except OSError as exc: if exc.errno == errno.EAGAIN: - raise JoyentMetadataTimeoutException(msg % as_ascii()) + raise JoyentMetadataTimeoutException( + msg % as_ascii() + ) from exc raise def _write(self, msg): diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 6156c75b..b968a96f 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -98,8 +98,10 @@ def get_boot_telemetry(): LOG.debug("Collecting boot telemetry") try: kernel_start = float(time.time()) - float(util.uptime()) - except ValueError: - raise RuntimeError("Failed to determine kernel start timestamp") + except ValueError as e: + raise RuntimeError( + "Failed to determine kernel start timestamp" + ) from e try: out, _ = subp.subp(['/bin/systemctl', @@ -116,12 +118,13 @@ def get_boot_telemetry(): user_start = kernel_start + (float(tsm) / 1000000) except subp.ProcessExecutionError as e: - raise RuntimeError("Failed to get UserspaceTimestampMonotonic: %s" - % e) + raise RuntimeError( + "Failed to get UserspaceTimestampMonotonic: %s" % e + ) from e except ValueError as e: - raise RuntimeError("Failed to parse " - "UserspaceTimestampMonotonic from systemd: %s" - % e) + raise RuntimeError( + "Failed to parse UserspaceTimestampMonotonic from systemd: %s" % e + ) from e try: out, _ = subp.subp(['/bin/systemctl', 'show', @@ -137,12 +140,14 @@ def get_boot_telemetry(): cloudinit_activation = kernel_start + (float(tsm) / 1000000) except subp.ProcessExecutionError as e: - raise RuntimeError("Failed to get InactiveExitTimestampMonotonic: %s" - % e) + raise RuntimeError( + "Failed to get InactiveExitTimestampMonotonic: %s" % e + ) from e except ValueError as e: - raise RuntimeError("Failed to parse " - "InactiveExitTimestampMonotonic from systemd: %s" - % e) + raise RuntimeError( + "Failed to parse InactiveExitTimestampMonotonic from systemd: %s" + % e + ) from e evt = events.ReportingEvent( BOOT_EVENT_TYPE, 'boot-telemetry', @@ -642,9 +647,10 @@ class WALinuxAgentShim: try: name = os.path.basename(hook_file).replace('.json', '') dhcp_options[name] = json.loads(util.load_file((hook_file))) - except ValueError: + except ValueError as e: raise ValueError( - '{_file} is not valid JSON data'.format(_file=hook_file)) + '{_file} is not valid JSON data'.format(_file=hook_file) + ) from e return dhcp_options @staticmethod diff --git a/cloudinit/sources/helpers/netlink.py b/cloudinit/sources/helpers/netlink.py index a74a3588..c2ad587b 100644 --- a/cloudinit/sources/helpers/netlink.py +++ b/cloudinit/sources/helpers/netlink.py @@ -74,7 +74,7 @@ def create_bound_netlink_socket(): netlink_socket.setblocking(0) except socket.error as e: msg = "Exception during netlink socket create: %s" % e - raise NetlinkCreateSocketError(msg) + raise NetlinkCreateSocketError(msg) from e LOG.debug("Created netlink socket") return netlink_socket diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 1050efb0..65e020c5 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -280,8 +280,9 @@ class BaseReader(metaclass=abc.ABCMeta): try: data = translator(data) except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to process path %s: %s" % (path, e) + ) from e if found: results[name] = data @@ -291,8 +292,9 @@ class BaseReader(metaclass=abc.ABCMeta): try: metadata['random_seed'] = base64.b64decode(random_seed) except (ValueError, TypeError) as e: - raise BrokenMetadata("Badly formatted metadata" - " random_seed entry: %s" % e) + raise BrokenMetadata( + "Badly formatted metadata random_seed entry: %s" % e + ) from e # load any files that were provided files = {} @@ -304,8 +306,9 @@ class BaseReader(metaclass=abc.ABCMeta): try: files[path] = self._read_content_path(item) except Exception as e: - raise BrokenMetadata("Failed to read provided " - "file %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to read provided file %s: %s" % (path, e) + ) from e results['files'] = files # The 'network_config' item in metadata is a content pointer @@ -317,8 +320,9 @@ class BaseReader(metaclass=abc.ABCMeta): content = self._read_content_path(net_item, decode=True) results['network_config'] = content except IOError as e: - raise BrokenMetadata("Failed to read network" - " configuration: %s" % (e)) + raise BrokenMetadata( + "Failed to read network configuration: %s" % (e) + ) from e # To openstack, user can specify meta ('nova boot --meta=key=value') # and those will appear under metadata['meta']. @@ -370,8 +374,9 @@ class ConfigDriveReader(BaseReader): try: return util.load_json(self._path_read(path)) except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to process path %s: %s" % (path, e) + ) from e def read_v1(self): """Reads a version 1 formatted location. @@ -395,16 +400,17 @@ class ConfigDriveReader(BaseReader): path = found[name] try: contents = self._path_read(path) - except IOError: - raise BrokenMetadata("Failed to read: %s" % path) + except IOError as e: + raise BrokenMetadata("Failed to read: %s" % path) from e try: # Disable not-callable pylint check; pylint isn't able to # determine that every member of FILES_V1 has a callable in # the appropriate position md[key] = translator(contents) # pylint: disable=E1102 except Exception as e: - raise BrokenMetadata("Failed to process " - "path %s: %s" % (path, e)) + raise BrokenMetadata( + "Failed to process path %s: %s" % (path, e) + ) from e else: md[key] = copy.deepcopy(default) diff --git a/cloudinit/subp.py b/cloudinit/subp.py index 804ef3ca..3e4efa42 100644 --- a/cloudinit/subp.py +++ b/cloudinit/subp.py @@ -262,7 +262,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True, raise ProcessExecutionError( cmd=args, reason=e, errno=e.errno, stdout="-" if decode else b"-", - stderr="-" if decode else b"-") + stderr="-" if decode else b"-" + ) from e finally: if devnull_fp: devnull_fp.close() diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index f3c0cf9c..caa88435 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -95,7 +95,7 @@ def read_file_or_url(url, **kwargs): code = e.errno if e.errno == ENOENT: code = NOT_FOUND - raise UrlError(cause=e, code=code, headers=None, url=url) + raise UrlError(cause=e, code=code, headers=None, url=url) from e return FileResponse(file_path, contents=contents) else: return readurl(url, **kwargs) @@ -575,8 +575,8 @@ def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret, timestamp=None): try: import oauthlib.oauth1 as oauth1 - except ImportError: - raise NotImplementedError('oauth support is not available') + except ImportError as e: + raise NotImplementedError('oauth support is not available') from e if timestamp: timestamp = str(timestamp) diff --git a/cloudinit/util.py b/cloudinit/util.py index dd263803..cf9e349f 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -359,7 +359,7 @@ def decomp_gzip(data, quiet=True, decode=True): if quiet: return data else: - raise DecompressionError(str(e)) + raise DecompressionError(str(e)) from e def extract_usergroup(ug_pair): @@ -1363,7 +1363,7 @@ def chownbyname(fname, user=None, group=None): if group: gid = grp.getgrnam(group).gr_gid except KeyError as e: - raise OSError("Unknown user or group: %s" % (e)) + raise OSError("Unknown user or group: %s" % (e)) from e chownbyid(fname, uid, gid) @@ -2387,8 +2387,8 @@ def human2bytes(size): try: num = float(num) - except ValueError: - raise ValueError("'%s' is not valid input." % size_in) + except ValueError as e: + raise ValueError("'%s' is not valid input." % size_in) from e if num < 0: raise ValueError("'%s': cannot be negative" % size_in) diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py index a136cf0d..eedbaae8 100644 --- a/tests/cloud_tests/platforms/azurecloud/instance.py +++ b/tests/cloud_tests/platforms/azurecloud/instance.py @@ -134,9 +134,10 @@ class AzureCloudInstance(Instance): self.vm_name, vm_params) LOG.debug('creating instance %s from image_id=%s', self.vm_name, self.image_id) - except CloudError: - raise RuntimeError('failed creating instance:\n{}'.format( - traceback.format_exc())) + except CloudError as e: + raise RuntimeError( + 'failed creating instance:\n{}'.format(traceback.format_exc()) + ) from e if wait: self.instance.wait() diff --git a/tests/cloud_tests/platforms/azurecloud/platform.py b/tests/cloud_tests/platforms/azurecloud/platform.py index cb62a74b..a664f612 100644 --- a/tests/cloud_tests/platforms/azurecloud/platform.py +++ b/tests/cloud_tests/platforms/azurecloud/platform.py @@ -59,9 +59,12 @@ class AzureCloudPlatform(Platform): self.vnet = self._create_vnet() self.subnet = self._create_subnet() self.nic = self._create_nic() - except CloudError: - raise RuntimeError('failed creating a resource:\n{}'.format( - traceback.format_exc())) + except CloudError as e: + raise RuntimeError( + 'failed creating a resource:\n{}'.format( + traceback.format_exc() + ) + ) from e def create_instance(self, properties, config, features, image_id, user_data=None): @@ -105,8 +108,10 @@ class AzureCloudPlatform(Platform): if image_id.find('__') > 0: image_id = image_id.split('__')[1] LOG.debug('image_id shortened to %s', image_id) - except KeyError: - raise RuntimeError('no images found for %s' % img_conf['release']) + except KeyError as e: + raise RuntimeError( + 'no images found for %s' % img_conf['release'] + ) from e return AzureCloudImage(self, img_conf, image_id) @@ -140,9 +145,11 @@ class AzureCloudPlatform(Platform): secret=azure_creds['clientSecret'], tenant=azure_creds['tenantId']) return credentials, subscription_id - except KeyError: - raise RuntimeError('Please configure Azure service principal' - ' credentials in %s' % cred_file) + except KeyError as e: + raise RuntimeError( + 'Please configure Azure service principal' + ' credentials in %s' % cred_file + ) from e def _create_resource_group(self): """Create resource group""" diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py index ab6037b1..d2e84047 100644 --- a/tests/cloud_tests/platforms/ec2/instance.py +++ b/tests/cloud_tests/platforms/ec2/instance.py @@ -49,11 +49,11 @@ class EC2Instance(Instance): # OutputBytes comes from platform._decode_console_output_as_bytes response = self.instance.console_output() return response['OutputBytes'] - except KeyError: + except KeyError as e: if 'Output' in response: msg = ("'OutputBytes' did not exist in console_output() but " "'Output' did: %s..." % response['Output'][0:128]) - raise util.PlatformError('console_log', msg) + raise util.PlatformError('console_log', msg) from e return ('No Console Output [%s]' % self.instance).encode() def destroy(self): diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py index 7a3d0fe0..b61a2ffb 100644 --- a/tests/cloud_tests/platforms/ec2/platform.py +++ b/tests/cloud_tests/platforms/ec2/platform.py @@ -35,12 +35,14 @@ class EC2Platform(Platform): self.ec2_resource = b3session.resource('ec2') self.ec2_region = b3session.region_name self.key_name = self._upload_public_key(config) - except botocore.exceptions.NoRegionError: + except botocore.exceptions.NoRegionError as e: raise RuntimeError( - 'Please configure default region in $HOME/.aws/config') - except botocore.exceptions.NoCredentialsError: + 'Please configure default region in $HOME/.aws/config' + ) from e + except botocore.exceptions.NoCredentialsError as e: raise RuntimeError( - 'Please configure ec2 credentials in $HOME/.aws/credentials') + 'Please configure ec2 credentials in $HOME/.aws/credentials' + ) from e self.vpc = self._create_vpc() self.internet_gateway = self._create_internet_gateway() @@ -125,8 +127,10 @@ class EC2Platform(Platform): try: image_ami = image['id'] - except KeyError: - raise RuntimeError('No images found for %s!' % img_conf['release']) + except KeyError as e: + raise RuntimeError( + 'No images found for %s!' % img_conf['release'] + ) from e LOG.debug('found image: %s', image_ami) image = EC2Image(self, img_conf, image_ami) @@ -195,7 +199,7 @@ class EC2Platform(Platform): CidrBlock=self.ipv4_cidr, AmazonProvidedIpv6CidrBlock=True) except botocore.exceptions.ClientError as e: - raise RuntimeError(e) + raise RuntimeError(e) from e vpc.wait_until_available() self._tag_resource(vpc) diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py index b27b9848..2b973a08 100644 --- a/tests/cloud_tests/platforms/lxd/instance.py +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -175,7 +175,8 @@ class LXDInstance(Instance): raise PlatformError( "console log", "Console log failed [%d]: stdout=%s stderr=%s" % ( - e.exit_code, e.stdout, e.stderr)) + e.exit_code, e.stdout, e.stderr) + ) from e def reboot(self, wait=True): """Reboot instance.""" diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py index 58f65e52..ac3b6563 100644 --- a/tests/cloud_tests/platforms/platforms.py +++ b/tests/cloud_tests/platforms/platforms.py @@ -74,8 +74,10 @@ class Platform(object): try: return tmirror.json_entries[0] - except IndexError: - raise RuntimeError('no images found with filter: %s' % img_filter) + except IndexError as e: + raise RuntimeError( + 'no images found with filter: %s' % img_filter + ) from e class FilterMirror(mirrors.BasicMirrorWriter): diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py index e8c371ca..bb9785d3 100644 --- a/tests/cloud_tests/testcases/__init__.py +++ b/tests/cloud_tests/testcases/__init__.py @@ -21,8 +21,10 @@ def discover_test(test_name): config.name_sanitize(test_name)) try: testmod = importlib.import_module(testmod_name) - except NameError: - raise ValueError('no test verifier found at: {}'.format(testmod_name)) + except NameError as e: + raise ValueError( + 'no test verifier found at: {}'.format(testmod_name) + ) from e found = [mod for name, mod in inspect.getmembers(testmod) if (inspect.isclass(mod) diff --git a/tools/mock-meta.py b/tools/mock-meta.py index a58e0260..9dd067b9 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -258,12 +258,14 @@ class MetaDataHandler(object): try: key_id = int(mybe_key) key_name = key_ids[key_id] - except ValueError: - raise WebException(hclient.BAD_REQUEST, - "%s: not an integer" % mybe_key) - except IndexError: - raise WebException(hclient.NOT_FOUND, - "Unknown key id %r" % mybe_key) + except ValueError as e: + raise WebException( + hclient.BAD_REQUEST, "%s: not an integer" % mybe_key + ) from e + except IndexError as e: + raise WebException( + hclient.NOT_FOUND, "Unknown key id %r" % mybe_key + ) from e # Extract the possible sub-params result = traverse(nparams[1:], { "openssh-key": "\n".join(avail_keys[key_name]), diff --git a/tox.ini b/tox.ini index f619dbf5..a92c63e0 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ setenv = basepython = python3 deps = # requirements - pylint==2.5.3 + pylint==2.6.0 # test-requirements because unit tests are now present in cloudinit tree -r{toxinidir}/test-requirements.txt -r{toxinidir}/integration-requirements.txt -- cgit v1.2.3